Compare commits
32 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4cec26edde | |||
| dc3879eca6 | |||
| 9669b8f477 | |||
| a7871ed7c6 | |||
| 79f5db2636 | |||
| 84146fc903 | |||
| c6f1fe317e | |||
| f46e5f5729 | |||
| 69fcc24a4d | |||
| 6a4f902e38 | |||
| 5f9c202d65 | |||
| 477454ec3c | |||
| 10812606bf | |||
| 94dec19976 | |||
| 9046f6e026 | |||
| 03504b61be | |||
| 6047d1a9f7 | |||
| 289eca8076 | |||
| 766f43931c | |||
| 14c24f8fda | |||
| e5916d522a | |||
| 4224dbe96c | |||
| 17ee290d90 | |||
| 7caafbaf80 | |||
| bc285799d1 | |||
| dcb2a86f9a | |||
| be739c319f | |||
| 28cf9faf77 | |||
| 279ef6cf8b | |||
| a602ff3efc | |||
| 79afd05ea5 | |||
| 42e846f876 |
@@ -0,0 +1,128 @@
|
|||||||
|
name: CI
|
||||||
|
|
||||||
|
# The release workflow gates a tag, which is late: a broken commit sits green until
|
||||||
|
# somebody decides to publish. This runs the same checks on the way in.
|
||||||
|
#
|
||||||
|
# push is scoped to main rather than all branches so that a branch pushed as part of a
|
||||||
|
# pull request is not checked twice.
|
||||||
|
#
|
||||||
|
# No actions/checkout, deliberately -- same as the letsvisit and charts workflows. The
|
||||||
|
# runner image is ubuntu:22.04 whose `nodejs` package is Node 12, and actions/checkout@v4
|
||||||
|
# is built with ES2022 static initialiser blocks, so it dies with
|
||||||
|
# `SyntaxError: Unexpected token '{'` before running. Cloning with git directly avoids JS
|
||||||
|
# actions entirely. This repo is public, so the clone needs no credential at all.
|
||||||
|
#
|
||||||
|
# `${{ }}` values are passed through `env:` and referenced as quoted shell variables: a
|
||||||
|
# ref name is attacker-influenced by anyone who can push a branch or open a PR, and
|
||||||
|
# expanding one straight into `run:` is a shell-injection vector.
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
# A rapid series of pushes only needs the last one checked.
|
||||||
|
concurrency:
|
||||||
|
group: ci-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
env:
|
||||||
|
REPO_URL: https://git.ryuvia.com/niklas/terdut-server.git
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container:
|
||||||
|
# Runs inside the toolchain image rather than installing Go per job. Note this puts
|
||||||
|
# the job on the dind bridge, which cannot reach github.com or get.helm.sh --
|
||||||
|
# proxy.golang.org and git.ryuvia.com are reachable, which is all this job needs.
|
||||||
|
image: golang:1.26.6-bookworm
|
||||||
|
# act_runner destroys a job's own volumes when it finishes, so without these every
|
||||||
|
# run re-downloads the whole module graph. The names must appear in the runner's
|
||||||
|
# container.valid_volumes allowlist (charts/act-runner in the k8s repo); unlisted
|
||||||
|
# volumes are dropped silently, so a workflow that looks correct can still be
|
||||||
|
# running uncached.
|
||||||
|
volumes:
|
||||||
|
- go-mod-cache:/go/pkg/mod
|
||||||
|
- go-build-cache:/root/.cache/go-build
|
||||||
|
- gobin-cache:/go/bin
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||||
|
run: |
|
||||||
|
if [ -n "$HEAD_SHA" ]; then
|
||||||
|
# A pull_request ref_name is "<n>/merge", which is not a fetchable branch.
|
||||||
|
git clone "$REPO_URL" .
|
||||||
|
git checkout -q "$HEAD_SHA"
|
||||||
|
else
|
||||||
|
git clone --depth=1 --branch "$REF_NAME" "$REPO_URL" .
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The gate is the Makefile's rather than a second copy of it here, the way riksdata
|
||||||
|
# and rd-web already do it. `make fmt lint test` is exactly what a developer runs, so
|
||||||
|
# a green pipeline and a green working copy mean the same thing by construction
|
||||||
|
# instead of by remembering to update two files together.
|
||||||
|
#
|
||||||
|
# The reasoning that used to live here moved with the targets: why gofmt is checked
|
||||||
|
# at all (import order survives `go vet`, and both repos sat unformatted through a
|
||||||
|
# green run and a release -- 9046f6e), why both of gofmt's failure modes need
|
||||||
|
# handling, and why `test` adds -race when this job does not have to.
|
||||||
|
- name: Format, vet and test
|
||||||
|
run: make fmt lint test
|
||||||
|
|
||||||
|
# Runs on every push and pull request, unlike the image scan, which needs something
|
||||||
|
# published to scan and so lives in release.yaml. Both are needed: govulncheck reads the
|
||||||
|
# source and its module graph, trivy reads the built artifact, and neither sees what the
|
||||||
|
# other does.
|
||||||
|
security:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container:
|
||||||
|
image: golang:1.26.6-bookworm
|
||||||
|
volumes:
|
||||||
|
- go-mod-cache:/go/pkg/mod
|
||||||
|
- go-build-cache:/root/.cache/go-build
|
||||||
|
- gobin-cache:/go/bin
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||||
|
run: |
|
||||||
|
if [ -n "$HEAD_SHA" ]; then
|
||||||
|
git clone "$REPO_URL" .
|
||||||
|
git checkout -q "$HEAD_SHA"
|
||||||
|
else
|
||||||
|
git clone --depth=1 --branch "$REF_NAME" "$REPO_URL" .
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Go vulnerability scan (govulncheck)
|
||||||
|
run: make security-go
|
||||||
|
|
||||||
|
- name: Secret scan (gitleaks)
|
||||||
|
run: make security-secrets
|
||||||
|
|
||||||
|
# Host mode, no `container:`: helm is baked into the runner image, and a container job
|
||||||
|
# could not install it -- get.helm.sh is unreachable from the dind bridge. Same reason
|
||||||
|
# release.yaml's chart job runs on the host.
|
||||||
|
#
|
||||||
|
# The chart had no lint step in any workflow until 2026-09-01: release.yaml packaged and
|
||||||
|
# pushed it without rendering it first, so a template that did not compile would have
|
||||||
|
# been found by Flux rather than here.
|
||||||
|
chart:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||||
|
run: |
|
||||||
|
if [ -n "$HEAD_SHA" ]; then
|
||||||
|
git clone "$REPO_URL" .
|
||||||
|
git checkout -q "$HEAD_SHA"
|
||||||
|
else
|
||||||
|
git clone --depth=1 --branch "$REF_NAME" "$REPO_URL" .
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Lint and render the chart
|
||||||
|
run: make helm-lint
|
||||||
@@ -0,0 +1,223 @@
|
|||||||
|
name: Release
|
||||||
|
|
||||||
|
# Checkout, interpolation and caching conventions match ci.yaml -- see the header there
|
||||||
|
# for why there are no JS actions and why every `${{ }}` goes through `env:`.
|
||||||
|
#
|
||||||
|
# There is no upload-artifact/download-artifact equivalent here (both are JS actions, and
|
||||||
|
# this Gitea has no artifact store wired up), so the job that builds the binaries is also
|
||||||
|
# the job that publishes them. Nothing is handed between jobs at all.
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- 'v*'
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
# A tag is not normally re-pushed, so this mostly matters when one is force-moved during
|
||||||
|
# a botched release -- the superseded run stops holding runner slots.
|
||||||
|
concurrency:
|
||||||
|
group: release-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
env:
|
||||||
|
REPO_URL: https://git.ryuvia.com/niklas/terdut-server.git
|
||||||
|
API: https://git.ryuvia.com/api/v1/repos/niklas/terdut-server
|
||||||
|
REGISTRY: git.ryuvia.com
|
||||||
|
IMAGE: git.ryuvia.com/niklas/terdut-server
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
# Gates every publishing job below. A tag that fails here publishes nothing: the
|
||||||
|
# binaries, the image and the chart are all downstream of it.
|
||||||
|
test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container:
|
||||||
|
image: golang:1.26.6-bookworm
|
||||||
|
volumes:
|
||||||
|
- go-mod-cache:/go/pkg/mod
|
||||||
|
- go-build-cache:/root/.cache/go-build
|
||||||
|
- gobin-cache:/go/bin
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
run: git clone --depth=1 --branch "$REF_NAME" "$REPO_URL" .
|
||||||
|
|
||||||
|
# Same gate as ci.yaml, and the same one a developer runs. See the Makefile for why
|
||||||
|
# each check is there; restating it here is how the two drift apart.
|
||||||
|
- name: Format, vet and test
|
||||||
|
run: make fmt lint test
|
||||||
|
|
||||||
|
binaries:
|
||||||
|
needs: test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container:
|
||||||
|
image: golang:1.26.6-bookworm
|
||||||
|
volumes:
|
||||||
|
- go-mod-cache:/go/pkg/mod
|
||||||
|
- go-build-cache:/root/.cache/go-build
|
||||||
|
- gobin-cache:/go/bin
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
run: git clone --depth=1 --branch "$REF_NAME" "$REPO_URL" .
|
||||||
|
|
||||||
|
# Compiling is the Makefile's; uploading is not. `make binaries` is runnable on a
|
||||||
|
# laptop, while the step below needs a token and the Gitea release API, which is
|
||||||
|
# this workflow's business and nothing a developer wants a target for.
|
||||||
|
- name: Build every target
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
run: make binaries VERSION="$REF_NAME"
|
||||||
|
|
||||||
|
# Creating the release is made idempotent rather than assumed-new: a re-run of a
|
||||||
|
# failed release must not die on the release that already exists. Assets are
|
||||||
|
# replaced the same way, so a re-run repairs a partial upload.
|
||||||
|
- name: Publish the release
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
auth="Authorization: token $TOKEN"
|
||||||
|
|
||||||
|
body=$(curl -sf -H "$auth" "$API/releases/tags/$REF_NAME" || true)
|
||||||
|
if [ -z "$body" ]; then
|
||||||
|
body=$(curl -sf -X POST -H "$auth" -H 'Content-Type: application/json' \
|
||||||
|
-d "{\"tag_name\":\"$REF_NAME\",\"name\":\"$REF_NAME\"}" \
|
||||||
|
"$API/releases")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The release object serialises `id` first, so the first match is the release's
|
||||||
|
# own id and not one of the nested author/asset ids.
|
||||||
|
release_id=$(printf '%s' "$body" | grep -o '"id":[0-9]*' | head -1 | cut -d: -f2)
|
||||||
|
[ -n "$release_id" ] || { echo "::error::could not determine release id"; exit 1; }
|
||||||
|
echo "release id $release_id"
|
||||||
|
|
||||||
|
for f in dist/*; do
|
||||||
|
name=$(basename "$f")
|
||||||
|
# Drop an existing asset of the same name first: Gitea happily stores two
|
||||||
|
# attachments with one name, and the updater matches by name.
|
||||||
|
old=$(curl -sf -H "$auth" "$API/releases/$release_id/assets" \
|
||||||
|
| tr '}' '\n' | grep "\"name\":\"$name\"" \
|
||||||
|
| grep -o '"id":[0-9]*' | head -1 | cut -d: -f2 || true)
|
||||||
|
if [ -n "$old" ]; then
|
||||||
|
curl -sf -X DELETE -H "$auth" "$API/releases/$release_id/assets/$old" || true
|
||||||
|
fi
|
||||||
|
echo "uploading $name"
|
||||||
|
curl -sf -X POST -H "$auth" -F "attachment=@$f" \
|
||||||
|
"$API/releases/$release_id/assets?name=$name" > /dev/null
|
||||||
|
done
|
||||||
|
|
||||||
|
# Host mode on purpose (no `container:`): this is the only context with a Docker CLI
|
||||||
|
# pointed at the dind daemon. A `container:` job would sit on the dind bridge with no
|
||||||
|
# docker socket at all.
|
||||||
|
image:
|
||||||
|
needs: test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
run: git clone --depth=1 --branch "$REF_NAME" "$REPO_URL" .
|
||||||
|
|
||||||
|
- name: Log in to the registry
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||||
|
run: echo "$TOKEN" | docker login "$REGISTRY" -u niklas --password-stdin
|
||||||
|
|
||||||
|
# buildx setup, the platform list and why there is no QEMU all live on the `push`
|
||||||
|
# target now, so the same command publishes from a laptop and from here.
|
||||||
|
- name: Build and push
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
run: make push VERSION="$REF_NAME"
|
||||||
|
|
||||||
|
# Also host mode: helm is baked into the runner image, and a `container:` job could not
|
||||||
|
# install it -- get.helm.sh is unreachable from the dind bridge.
|
||||||
|
chart:
|
||||||
|
needs: test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
run: git clone --depth=1 --branch "$REF_NAME" "$REPO_URL" .
|
||||||
|
|
||||||
|
# This job is the only thing that publishes the chart, which is what keeps the
|
||||||
|
# published metadata honest. There used to be a second publisher on every charts/**
|
||||||
|
# push to main, and the two raced for the same chart version with different answers:
|
||||||
|
# this one stamps version and appVersion from the tag, that one took Chart.yaml
|
||||||
|
# verbatim, where appVersion is the hardcoded "latest". Whichever landed first won,
|
||||||
|
# so the metadata of a release depended on which runner was quicker -- chart 0.9.0
|
||||||
|
# went out on 2026-08-08 reading appVersion "latest" that way.
|
||||||
|
#
|
||||||
|
# It could not be fixed by making both agree: the tag is pushed after the branch, so
|
||||||
|
# a workflow triggered by the main push cannot know the version it is about to be
|
||||||
|
# tagged with. One publisher, triggered by the tag.
|
||||||
|
#
|
||||||
|
# The cost is that the chart only ships with an app release. That is no real loss --
|
||||||
|
# `make helm-package` derives the chart version from the tag, so a chart-only change
|
||||||
|
# has no version of its own to be released under anyway. Chart fixes ride the next
|
||||||
|
# tag.
|
||||||
|
- name: Refuse a non-version tag
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
if ! echo "$REF_NAME" | grep -qE '^v[0-9]'; then
|
||||||
|
echo "::error::refusing to publish a chart for non-version tag ${REF_NAME}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Render before publishing. Until 2026-09-01 this job packaged and pushed without
|
||||||
|
# linting, so a template that did not compile reached the registry and was found by
|
||||||
|
# Flux instead.
|
||||||
|
- name: Lint and render the chart
|
||||||
|
run: make helm-lint
|
||||||
|
|
||||||
|
# The version and appVersion are no longer sed'd into Chart.yaml before packaging:
|
||||||
|
# `helm package --version --app-version` sets both from the tag without mutating the
|
||||||
|
# tree mid-build, which is what the rest of the release process already assumed
|
||||||
|
# happened. The isolated helm repo list moved onto the targets with them.
|
||||||
|
- name: Package and push
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
echo "$TOKEN" | helm registry login "$REGISTRY" -u niklas --password-stdin
|
||||||
|
make helm-package helm-push VERSION="$REF_NAME"
|
||||||
|
|
||||||
|
# Host mode, like image and chart: this needs a docker daemon to run trivy in, and a
|
||||||
|
# `container:` job would sit on the dind bridge with none.
|
||||||
|
#
|
||||||
|
# It scans the pushed image rather than a locally built one, because trivy cannot read a
|
||||||
|
# local image on this runner -- Talos has no docker socket and the dind sidecar shares no
|
||||||
|
# filesystem with the job -- so it pulls from the registry. That is also why this runs
|
||||||
|
# after `image` rather than gating it: a red scan does not unpublish anything.
|
||||||
|
#
|
||||||
|
# What a red scan means is therefore not "the release failed" but "do not bump the wrapper
|
||||||
|
# chart in Ryuvia/charts to this version". The image and chart are already published by
|
||||||
|
# the time this runs, and deliberately so -- this pipeline does not deploy.
|
||||||
|
#
|
||||||
|
# riksdata and rd-web have had this since they were set up; terdut-server went without any
|
||||||
|
# image scanning until 2026-09-02, so every release before v0.9.4 was published with no
|
||||||
|
# CVE check at all.
|
||||||
|
scan-image:
|
||||||
|
needs: image
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
env:
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
run: git clone --depth=1 --branch "$REF_NAME" "$REPO_URL" .
|
||||||
|
|
||||||
|
# Credentials are passed even though these packages are anonymously pullable -- that
|
||||||
|
# is a property of the personal namespace this publishes to, not something a release
|
||||||
|
# should depend on staying true.
|
||||||
|
- name: Scan the pushed image (trivy)
|
||||||
|
env:
|
||||||
|
TRIVY_USERNAME: niklas
|
||||||
|
TRIVY_PASSWORD: ${{ secrets.REGISTRY_TOKEN }}
|
||||||
|
REF_NAME: ${{ github.ref_name }}
|
||||||
|
run: make security-image VERSION="$REF_NAME"
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
name: Release Helm Chart
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
paths:
|
|
||||||
- charts/**
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
release:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
pages: write
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
- name: Configure Git
|
|
||||||
run: |
|
|
||||||
git config user.name "$GITHUB_ACTOR"
|
|
||||||
git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
|
|
||||||
|
|
||||||
- name: Install Helm
|
|
||||||
uses: azure/setup-helm@v4
|
|
||||||
|
|
||||||
- name: Run chart-releaser
|
|
||||||
uses: helm/chart-releaser-action@v1.6.0
|
|
||||||
env:
|
|
||||||
CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
|
||||||
@@ -1,123 +0,0 @@
|
|||||||
name: Release
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
tags:
|
|
||||||
- 'v*'
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
strategy:
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
- goos: linux
|
|
||||||
goarch: amd64
|
|
||||||
- goos: linux
|
|
||||||
goarch: arm64
|
|
||||||
- goos: darwin
|
|
||||||
goarch: amd64
|
|
||||||
- goos: darwin
|
|
||||||
goarch: arm64
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- uses: actions/setup-go@v5
|
|
||||||
with:
|
|
||||||
go-version-file: go.mod
|
|
||||||
|
|
||||||
- name: Build
|
|
||||||
env:
|
|
||||||
GOOS: ${{ matrix.goos }}
|
|
||||||
GOARCH: ${{ matrix.goarch }}
|
|
||||||
run: |
|
|
||||||
go build \
|
|
||||||
-ldflags "-w -s -X main.version=${{ github.ref_name }}" \
|
|
||||||
-o terdut-${{ github.ref_name }}-${{ matrix.goos }}-${{ matrix.goarch }} \
|
|
||||||
./cmd/terdut
|
|
||||||
|
|
||||||
- uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: terdut-${{ github.ref_name }}-${{ matrix.goos }}-${{ matrix.goarch }}
|
|
||||||
path: terdut-${{ github.ref_name }}-${{ matrix.goos }}-${{ matrix.goarch }}
|
|
||||||
|
|
||||||
docker:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
packages: write
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Set up QEMU
|
|
||||||
uses: docker/setup-qemu-action@v3
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v3
|
|
||||||
|
|
||||||
- name: Log in to GHCR
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
with:
|
|
||||||
registry: ghcr.io
|
|
||||||
username: ${{ github.actor }}
|
|
||||||
password: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
|
|
||||||
- name: Build and push
|
|
||||||
uses: docker/build-push-action@v6
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
platforms: linux/amd64,linux/arm64
|
|
||||||
push: true
|
|
||||||
build-args: VERSION=${{ github.ref_name }}
|
|
||||||
tags: |
|
|
||||||
ghcr.io/yeniklas/terdut-server:latest
|
|
||||||
ghcr.io/yeniklas/terdut-server:${{ github.ref_name }}
|
|
||||||
|
|
||||||
chart:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
- name: Configure Git
|
|
||||||
run: |
|
|
||||||
git config user.name "$GITHUB_ACTOR"
|
|
||||||
git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
|
|
||||||
|
|
||||||
- name: Install Helm
|
|
||||||
uses: azure/setup-helm@v4
|
|
||||||
|
|
||||||
- name: Update chart versions
|
|
||||||
run: |
|
|
||||||
VERSION="${{ github.ref_name }}"
|
|
||||||
if [[ "$VERSION" =~ ^v[0-9] ]]; then
|
|
||||||
CHART_VERSION="${VERSION#v}"
|
|
||||||
sed -i "s/^version:.*/version: ${CHART_VERSION}/" charts/terdut-server/Chart.yaml
|
|
||||||
sed -i "s/^appVersion:.*/appVersion: \"${VERSION}\"/" charts/terdut-server/Chart.yaml
|
|
||||||
fi
|
|
||||||
|
|
||||||
- name: Run chart-releaser
|
|
||||||
uses: helm/chart-releaser-action@v1.6.0
|
|
||||||
with:
|
|
||||||
skip_existing: true
|
|
||||||
env:
|
|
||||||
CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
|
|
||||||
|
|
||||||
release:
|
|
||||||
needs: build
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
contents: write
|
|
||||||
steps:
|
|
||||||
- uses: actions/download-artifact@v4
|
|
||||||
with:
|
|
||||||
merge-multiple: true
|
|
||||||
|
|
||||||
- uses: softprops/action-gh-release@v2
|
|
||||||
with:
|
|
||||||
files: 'terdut-*'
|
|
||||||
@@ -1,6 +1,11 @@
|
|||||||
# build output
|
# build output
|
||||||
/terdut
|
/terdut
|
||||||
/terdut-server
|
/terdut-server
|
||||||
|
# `make binaries` and `make helm-package` write here
|
||||||
|
/dist/
|
||||||
|
# isolated helm repo list written by the publishing targets, so the machine-wide
|
||||||
|
# one (which has an unreachable entry) cannot abort a release
|
||||||
|
/.helm-repos.yaml
|
||||||
|
|
||||||
# SQLite database files
|
# SQLite database files
|
||||||
*.db
|
*.db
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# Read by the `release` skill (~/.claude/skills/release).
|
||||||
|
#
|
||||||
|
# Only what the Makefile cannot already say. IMAGE, HELM_CHART and HELM_REPO come from
|
||||||
|
# `make release-vars`, so they have one definition and cannot drift from what is built.
|
||||||
|
#
|
||||||
|
# Defaults, set here only where this repo differs:
|
||||||
|
# CHARTS_REPO=$HOME/git/charts CHARTS_DIR=<image basename>
|
||||||
|
# GITEA_LOGIN=Ryuvia APPVERSION_PREFIX=
|
||||||
|
# PROSE_LANG=en
|
||||||
|
|
||||||
|
# Same as the image basename, so this is only stated to be read rather than derived.
|
||||||
|
CHARTS_DIR=terdut-server
|
||||||
|
|
||||||
|
# riksdata writes appVersion: "v0.3.1", rd-web writes a bare 0.5.0; this repo writes the
|
||||||
|
# v, like riksdata. Nothing reads the field -- .gitea/workflows/release.yaml stamps both
|
||||||
|
# version and appVersion from the tag when it publishes -- but people read it, and until
|
||||||
|
# 2026-09-01 it said "latest" while the tree headed for a numbered release.
|
||||||
|
APPVERSION_PREFIX=v
|
||||||
|
|
||||||
|
# English. The Swedish in riksdata and rd-web follows from their subject matter, not from a
|
||||||
|
# house style: terdut-server is an on-call tool whose labels, API and data are English, and
|
||||||
|
# nothing about it is coupled to Swedish. Code comments and docs here were always English;
|
||||||
|
# from 2026-09-01 the release prose is too.
|
||||||
|
PROSE_LANG=en
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
## Release
|
||||||
|
|
||||||
|
Say **"Release"** (or "Release X.Y.Z") and the `release` skill runs it: commit, push, tag,
|
||||||
|
wait for the pipeline, then open the wrapper-chart PR against `Ryuvia/charts`. It stops
|
||||||
|
there — merging and the Flux reconcile stay manual, deliberately.
|
||||||
|
|
||||||
|
Preconditions and the plan, without side effects:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
~/.claude/skills/release/scripts/release-preflight # state + suggested version
|
||||||
|
~/.claude/skills/release/scripts/release-preflight vX.Y.Z # validate that release
|
||||||
|
```
|
||||||
|
|
||||||
|
Config is `.release.conf` here plus `make release-vars`. The process itself lives in
|
||||||
|
`~/.claude/skills/release/`; why it is shaped this way is in README.md §Releasing.
|
||||||
|
|
||||||
|
Two things about this repo specifically:
|
||||||
|
|
||||||
|
- **The image is scanned after it is published, not before.** `scan-image` runs trivy
|
||||||
|
against the pushed image, because trivy cannot read a locally built one on this runner.
|
||||||
|
A red scan therefore unpublishes nothing — it means: do not bump the wrapper chart in
|
||||||
|
`Ryuvia/charts` to this version. Added 2026-09-02; every release up to and including
|
||||||
|
v0.9.3 was published with no CVE check at all.
|
||||||
|
- **The wrapper chart has two `tag:` lines** — the app image and the python backup sidecar —
|
||||||
|
so `chart-bump` needs `--image "$IMAGE"` to know which one moves.
|
||||||
|
|
||||||
|
## Checks
|
||||||
|
|
||||||
|
`make fmt lint test helm-lint` **is** what the pipeline runs — `ci.yaml` and `release.yaml`
|
||||||
|
call these targets rather than restating them, the way riksdata and rd-web do. A green gate
|
||||||
|
here and a green pipeline are the same code, not two descriptions of it. `test` adds `-race`,
|
||||||
|
which the workflows do not have to ask for since they call the target; see the comment on it
|
||||||
|
for why.
|
||||||
|
|
||||||
|
`make release` (build + push the multi-arch image, package + push the chart) is what
|
||||||
|
`release.yaml` invokes. Do not run it by hand — it refuses `VERSION=dev` for that reason, and
|
||||||
|
publishing happens by pushing a tag.
|
||||||
|
|
||||||
|
Three scans, and they see different things: `security-go` (govulncheck) reads the source and
|
||||||
|
its module graph and reports only vulnerabilities the code can actually reach;
|
||||||
|
`security-secrets` (gitleaks) reads the working tree, not the history, so it catches a secret
|
||||||
|
on the way in rather than auditing what is already committed; `security-image` (trivy) reads
|
||||||
|
the published artifact and therefore only runs on a tag. The first two gate every push.
|
||||||
+11
-2
@@ -1,10 +1,19 @@
|
|||||||
FROM golang:1.25-alpine AS builder
|
# --platform=$BUILDPLATFORM pins the builder to the machine doing the building, so a
|
||||||
|
# multi-arch build compiles both targets natively instead of running an emulated arm64
|
||||||
|
# toolchain under QEMU. Go cross-compiles from TARGETOS/TARGETARCH, which BuildKit fills
|
||||||
|
# in per platform. The CI runner has no binfmt registration and no way to get one (the
|
||||||
|
# JS action that used to install it cannot run there), so this is not just an
|
||||||
|
# optimisation -- it is what makes the arm64 image buildable at all.
|
||||||
|
FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS builder
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
COPY . .
|
COPY . .
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s -X main.version=${VERSION}" -o /terdut ./cmd/terdut
|
ARG TARGETOS
|
||||||
|
ARG TARGETARCH
|
||||||
|
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||||
|
go build -ldflags="-w -s -X main.version=${VERSION}" -o /terdut ./cmd/terdut
|
||||||
|
|
||||||
FROM scratch
|
FROM scratch
|
||||||
COPY --from=builder /terdut /terdut
|
COPY --from=builder /terdut /terdut
|
||||||
|
|||||||
@@ -0,0 +1,219 @@
|
|||||||
|
REGISTRY := git.ryuvia.com
|
||||||
|
# The personal namespace, not ryuvia — deliberately, and for one reason: Gitea
|
||||||
|
# scopes package visibility to the owner with no per-package override, so
|
||||||
|
# ryuvia/* is private because the org is. Publishing here keeps the image and
|
||||||
|
# chart anonymously pullable, so no pull secret is needed in the cluster and
|
||||||
|
# Flux needs no registry credentials. Same choice riksdata and rd-web made.
|
||||||
|
OWNER := niklas
|
||||||
|
|
||||||
|
IMAGE := $(REGISTRY)/$(OWNER)/terdut-server
|
||||||
|
HELM_CHART := charts/terdut-server
|
||||||
|
HELM_REPO := oci://$(REGISTRY)/$(OWNER)
|
||||||
|
|
||||||
|
# go.mod pins an exact patch release so nobody builds the shipped binary with a
|
||||||
|
# toolchain carrying known stdlib CVEs. Fedora's Go package overrides the
|
||||||
|
# upstream GOTOOLCHAIN default to `local`, which turns that pin into a hard
|
||||||
|
# failure on a dev box one patch behind, so restore the upstream default here.
|
||||||
|
export GOTOOLCHAIN ?= auto
|
||||||
|
|
||||||
|
.PHONY: help
|
||||||
|
help: ## Show this help
|
||||||
|
@grep -hE '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | \
|
||||||
|
awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-22s\033[0m %s\n", $$1, $$2}'
|
||||||
|
|
||||||
|
## --- checks ---
|
||||||
|
#
|
||||||
|
# These three mirror .gitea/workflows/ci.yaml step for step, so a green `make fmt
|
||||||
|
# lint test` here means the same thing CI means. The one deliberate difference is
|
||||||
|
# -race below.
|
||||||
|
|
||||||
|
.PHONY: test
|
||||||
|
test: ## Run the test suite
|
||||||
|
go test -race ./...
|
||||||
|
|
||||||
|
# CI runs a bare `go test ./...`. This is stricter on purpose: the sweeper, the
|
||||||
|
# notifier goroutine and the deadman sweep all touch the same single-connection
|
||||||
|
# database, and a race there would surface as a flaky production incident rather
|
||||||
|
# than a failed build. It passes today; if it ever costs more than it catches,
|
||||||
|
# the honest fix is to teach CI -race too, not to quietly drop it here.
|
||||||
|
.PHONY: lint
|
||||||
|
lint: ## go vet
|
||||||
|
go vet ./...
|
||||||
|
|
||||||
|
# Copied from ci.yaml rather than simplified, because both of gofmt's failure
|
||||||
|
# modes need handling and they are not alike. A file that is merely misformatted
|
||||||
|
# is listed on stdout with exit 0 — so the failure has to be raised by hand. A
|
||||||
|
# file that does not parse is the opposite: nothing on stdout and exit 2, which a
|
||||||
|
# naive `[ -n "$$out" ]` reads as success. See 9046f6e.
|
||||||
|
.PHONY: fmt
|
||||||
|
fmt: ## Report unformatted files
|
||||||
|
@if ! unformatted=$$(gofmt -l .); then \
|
||||||
|
echo "gofmt could not parse the tree:"; gofmt -l .; exit 1; \
|
||||||
|
fi; \
|
||||||
|
if [ -n "$$unformatted" ]; then \
|
||||||
|
echo "gofmt needed:"; echo "$$unformatted"; gofmt -d .; exit 1; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
.PHONY: helm-lint
|
||||||
|
helm-lint: ## Lint and render the chart
|
||||||
|
helm lint $(HELM_CHART) --set image.tag=v0.0.0
|
||||||
|
helm template terdut-server $(HELM_CHART) --namespace terdut-server \
|
||||||
|
--set image.tag=v0.0.0 >/dev/null
|
||||||
|
@# networking.listener defaults to "", which attaches the route to every
|
||||||
|
@# matching listener including plaintext HTTP. Production sets it, so the
|
||||||
|
@# default render proves nothing about the path that actually ships.
|
||||||
|
helm template terdut-server $(HELM_CHART) --namespace terdut-server \
|
||||||
|
--set image.tag=v0.0.0 --set networking.listener=https-terdut >/dev/null
|
||||||
|
|
||||||
|
## --- release ---
|
||||||
|
|
||||||
|
# The release process (~/.claude/skills/release) reads these rather than restating them.
|
||||||
|
# One definition, so the version that gets tagged, the image that gets pushed and the chart
|
||||||
|
# the wrapper pins cannot drift apart in a second copy.
|
||||||
|
.PHONY: release-vars
|
||||||
|
release-vars: ## Print the variables the release process reads
|
||||||
|
@printf 'IMAGE=%s\nHELM_CHART=%s\nHELM_REPO=%s\n' '$(IMAGE)' '$(HELM_CHART)' '$(HELM_REPO)'
|
||||||
|
|
||||||
|
# There is deliberately no build/push/helm-package/helm-push/release here, unlike
|
||||||
|
# riksdata and rd-web. .gitea/workflows/release.yaml owns publishing for this repo,
|
||||||
|
# and it does two things a local make cannot: it builds linux/amd64 and linux/arm64
|
||||||
|
# through buildx, and it stamps the chart's version and appVersion from the tag. A
|
||||||
|
# `docker build && docker push` target would push a single-architecture image over
|
||||||
|
# the multi-arch tag, which is both easy to do by accident and invisible afterwards
|
||||||
|
# — the tag would still resolve, just not on arm64. Publishing happens by pushing a
|
||||||
|
# tag; nothing else.
|
||||||
|
|
||||||
|
## --- publishing ---
|
||||||
|
#
|
||||||
|
# These exist so .gitea/workflows/release.yaml can call `make release` instead of
|
||||||
|
# restating the build in YAML, the way riksdata and rd-web already do. One definition
|
||||||
|
# of how this is built and published, runnable locally, reviewable in a diff.
|
||||||
|
#
|
||||||
|
# VERSION is the git tag, passed in by the workflow. The guard below is why a stray
|
||||||
|
# local `make release` cannot publish: dev is not a version anyone releases.
|
||||||
|
|
||||||
|
VERSION ?= dev
|
||||||
|
|
||||||
|
# Helm requires strict SemVer — strip a leading 'v' if present.
|
||||||
|
CHART_VERSION := $(shell echo "$(VERSION)" | sed 's/^v//')
|
||||||
|
|
||||||
|
PLATFORMS ?= linux/amd64,linux/arm64
|
||||||
|
BUILDX_BUILDER ?= terdut
|
||||||
|
|
||||||
|
TRIVY_VERSION := 0.73.0
|
||||||
|
GOVULNCHECK_VERSION := v1.1.4
|
||||||
|
GITLEAKS_VERSION := v8.30.0
|
||||||
|
|
||||||
|
# --pull, not --no-cache: refresh the base image without discarding the layer cache.
|
||||||
|
DOCKER_BUILD_FLAGS ?= --pull
|
||||||
|
|
||||||
|
# An isolated repo list. The machine-wide one is not this build's business, and one
|
||||||
|
# unreachable entry in it aborts otherwise-fine helm commands — there is a dead
|
||||||
|
# TrueCharts repo on this host that does exactly that. HELM_REPOSITORY_CACHE is
|
||||||
|
# deliberately NOT overridden alongside it: helm writes a refreshed index to the default
|
||||||
|
# cache and then looks for it in the overridden one.
|
||||||
|
HELM_ISOLATED = HELM_REPOSITORY_CONFIG=$(CURDIR)/.helm-repos.yaml
|
||||||
|
|
||||||
|
.PHONY: require-version
|
||||||
|
require-version:
|
||||||
|
@test "$(VERSION)" != "dev" || \
|
||||||
|
(echo "VERSION=dev names no release — pass VERSION=vX.Y.Z (the workflow passes the tag)" && exit 1)
|
||||||
|
|
||||||
|
.PHONY: build
|
||||||
|
build: ## Build the image for this host only, without pushing (local check / CI smoke)
|
||||||
|
docker build $(DOCKER_BUILD_FLAGS) \
|
||||||
|
--build-arg VERSION=$(VERSION) \
|
||||||
|
-t $(IMAGE):$(VERSION) .
|
||||||
|
|
||||||
|
# Multi-arch, so unlike riksdata and rd-web this cannot be a separate build then push:
|
||||||
|
# buildx cannot load a multi-platform result into the local image store, it can only
|
||||||
|
# push it. `build` above stays single-platform and local-only for that reason.
|
||||||
|
#
|
||||||
|
# No QEMU: the Dockerfile's builder stage runs on $$BUILDPLATFORM and cross-compiles from
|
||||||
|
# TARGETARCH, so both platforms build natively. The default "docker" driver cannot build
|
||||||
|
# more than one platform at a time; the docker-container driver can.
|
||||||
|
.PHONY: push
|
||||||
|
push: require-version ## Build and publish the multi-arch image
|
||||||
|
docker buildx create --name $(BUILDX_BUILDER) --use 2>/dev/null || docker buildx use $(BUILDX_BUILDER)
|
||||||
|
docker buildx build \
|
||||||
|
--platform $(PLATFORMS) \
|
||||||
|
--build-arg "VERSION=$(VERSION)" \
|
||||||
|
--tag "$(IMAGE):latest" \
|
||||||
|
--tag "$(IMAGE):$(VERSION)" \
|
||||||
|
--push .
|
||||||
|
|
||||||
|
# --version and --app-version come from the tag, so Chart.yaml's own fields decide nothing
|
||||||
|
# about what is published. They used to be rewritten in place with sed before packaging;
|
||||||
|
# the flags do the same job without mutating the tree mid-build.
|
||||||
|
.PHONY: helm-package
|
||||||
|
helm-package: require-version ## Package the chart, versioned from the tag
|
||||||
|
$(HELM_ISOLATED) helm package $(HELM_CHART) \
|
||||||
|
--version $(CHART_VERSION) \
|
||||||
|
--app-version $(VERSION) \
|
||||||
|
--destination dist
|
||||||
|
|
||||||
|
.PHONY: helm-push
|
||||||
|
helm-push: require-version ## Push the packaged chart to the OCI registry
|
||||||
|
$(HELM_ISOLATED) helm push dist/terdut-server-$(CHART_VERSION).tgz $(HELM_REPO)
|
||||||
|
|
||||||
|
.PHONY: binaries
|
||||||
|
binaries: require-version ## Cross-compile the release binaries into dist/
|
||||||
|
@mkdir -p dist
|
||||||
|
@set -eu; for target in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64; do \
|
||||||
|
GOOS="$${target%/*}"; GOARCH="$${target#*/}"; \
|
||||||
|
out="dist/terdut-$(VERSION)-$${GOOS}-$${GOARCH}"; \
|
||||||
|
echo "building $$out"; \
|
||||||
|
GOOS="$$GOOS" GOARCH="$$GOARCH" go build \
|
||||||
|
-ldflags "-w -s -X main.version=$(VERSION)" \
|
||||||
|
-o "$$out" ./cmd/terdut; \
|
||||||
|
done
|
||||||
|
|
||||||
|
.PHONY: release
|
||||||
|
release: push helm-package helm-push ## Publish image + chart (the workflow's one call)
|
||||||
|
|
||||||
|
## --- security ---
|
||||||
|
|
||||||
|
# Symbol-level, not dependency-level: govulncheck reports a vulnerability only when the
|
||||||
|
# code can actually reach it. As of 2026-09-02 this repo imports three chi advisories and
|
||||||
|
# reports none of them, because all three are middleware.RealIP and router.go uses Logger
|
||||||
|
# and Recoverer. That is the useful property rather than a loophole -- adding
|
||||||
|
# middleware.RealIP would turn this red, which is exactly when someone should look.
|
||||||
|
.PHONY: security-go
|
||||||
|
security-go: ## Scan Go deps for known CVEs (govulncheck)
|
||||||
|
go run golang.org/x/vuln/cmd/govulncheck@$(GOVULNCHECK_VERSION) ./...
|
||||||
|
|
||||||
|
# --no-git scans the working tree rather than the history, so this catches a secret on the
|
||||||
|
# way in. It is not a history audit and finding nothing here says nothing about what is
|
||||||
|
# already committed. --redact because the finding is printed into a CI log.
|
||||||
|
#
|
||||||
|
# Note when testing it that gitleaks allowlists well-known example credentials -- the AWS
|
||||||
|
# key from their own documentation does not trip it. A private key block does.
|
||||||
|
.PHONY: security-secrets
|
||||||
|
security-secrets: ## Scan the working tree for committed secrets (gitleaks)
|
||||||
|
go run github.com/zricethezav/gitleaks/v8@$(GITLEAKS_VERSION) detect --no-git \
|
||||||
|
--source . --redact --no-banner --exit-code 1
|
||||||
|
|
||||||
|
|
||||||
|
# Scans the pushed image, not a local one: trivy cannot read a locally built image on the
|
||||||
|
# runner -- Talos has no docker socket, and the dind sidecar shares no filesystem with the
|
||||||
|
# job -- so it pulls from the registry. Same reason riksdata and rd-web scan after pushing.
|
||||||
|
#
|
||||||
|
# It cannot gate a deploy, because this pipeline does not deploy. A red scan means: do not
|
||||||
|
# bump the wrapper chart in Ryuvia/charts to this version.
|
||||||
|
#
|
||||||
|
# The image is FROM scratch, so there are no OS packages to scan and trivy sees exactly one
|
||||||
|
# target -- the Go binary and its module graph. That also makes scanning a single platform
|
||||||
|
# sufficient here: linux/amd64 and linux/arm64 are the same modules built for a different
|
||||||
|
# GOARCH, so a CVE in one is a CVE in both. On an image with a base layer that would not
|
||||||
|
# hold and both platforms would need scanning.
|
||||||
|
#
|
||||||
|
# This is the last of the three scans and the only one that needs a published artifact;
|
||||||
|
# security-go and security-secrets above run on every push.
|
||||||
|
.PHONY: security-image
|
||||||
|
security-image: require-version ## Scan the pushed image for CVEs (needs VERSION)
|
||||||
|
@# The named volume persists trivy's vulnerability DB between runs; without it every
|
||||||
|
@# scan re-downloads the whole database.
|
||||||
|
docker run --rm -e TRIVY_USERNAME -e TRIVY_PASSWORD \
|
||||||
|
-v trivy-cache:/root/.cache/trivy \
|
||||||
|
docker.io/aquasec/trivy:$(TRIVY_VERSION) image --severity HIGH,CRITICAL \
|
||||||
|
--ignore-unfixed --exit-code 1 $(IMAGE):$(VERSION)
|
||||||
@@ -1,11 +1,13 @@
|
|||||||
# Terminal Duty (terdut-server)
|
# Terminal Duty (terdut-server)
|
||||||
|
|
||||||
On-call alert management server for teams using Prometheus Alertmanager.
|
Incident management server for teams using Prometheus Alertmanager.
|
||||||
|
|
||||||
- Receives Alertmanager webhooks directly — no adapter needed
|
- Receives Alertmanager webhooks directly — no adapter needed
|
||||||
- Stores and queries alerts (acknowledge, comment)
|
- Turns alerts into **incidents**, correlated by Alertmanager's own `groupKey`
|
||||||
- On-call schedule management (user-to-day assignments)
|
- Incident workflow: acknowledge, assign, snooze, note, resolve, with a full timeline
|
||||||
- Alert statistics (by status, by hour, by day)
|
- On-call schedule management, with new incidents auto-assigned to whoever is on call
|
||||||
|
- Alert and incident statistics, including MTTA and MTTR
|
||||||
|
- Web UI for phones and desktops, served by the same binary
|
||||||
- REST API with per-user API key authentication
|
- REST API with per-user API key authentication
|
||||||
- Single binary, SQLite storage — trivial to self-host
|
- Single binary, SQLite storage — trivial to self-host
|
||||||
|
|
||||||
@@ -16,7 +18,7 @@ On-call alert management server for teams using Prometheus Alertmanager.
|
|||||||
**Prerequisites:** Go 1.21+
|
**Prerequisites:** Go 1.21+
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://github.com/yeniklas/terdut-server
|
git clone https://git.ryuvia.com/niklas/terdut-server
|
||||||
cd terdut-server
|
cd terdut-server
|
||||||
go run ./cmd/terdut
|
go run ./cmd/terdut
|
||||||
```
|
```
|
||||||
@@ -28,10 +30,11 @@ The server starts on `:8080` with a `terdut.db` file in the working directory.
|
|||||||
```bash
|
```bash
|
||||||
curl -X POST http://localhost:8080/api/bootstrap \
|
curl -X POST http://localhost:8080/api/bootstrap \
|
||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
-d '{"username": "admin", "email": "admin@example.com"}'
|
-d '{"username": "admin", "email": "admin@example.com", "password": "<at least 10 characters>"}'
|
||||||
```
|
```
|
||||||
|
|
||||||
Save the `api_key.key` value from the response — it is shown **once only**.
|
Save the `api_key.key` value from the response — it is shown **once only**. The
|
||||||
|
`password` is optional and is what signs you in to the [web UI](#web-ui).
|
||||||
|
|
||||||
Use it as a bearer token for all subsequent requests:
|
Use it as a bearer token for all subsequent requests:
|
||||||
|
|
||||||
@@ -40,6 +43,48 @@ export KEY=<your-key>
|
|||||||
curl -H "Authorization: Bearer $KEY" http://localhost:8080/api/users
|
curl -H "Authorization: Bearer $KEY" http://localhost:8080/api/users
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Web UI
|
||||||
|
|
||||||
|
The server serves a web UI at `/`: the incident queue, each incident's alerts
|
||||||
|
and timeline with every action (acknowledge, assign, snooze, note, resolve,
|
||||||
|
archive), who is on call, the alert feed, and changing your own password. It is
|
||||||
|
built for a phone first. On a phone it has a bottom tab bar and a sticky action
|
||||||
|
bar, it follows the system's dark mode, and it can be added to the home screen.
|
||||||
|
From 900px wide it switches to a sidebar with the queue and the incident side by
|
||||||
|
side. Schedule editing, statistics and user management remain in
|
||||||
|
[terdut-tui](https://github.com/yeniklas/terdut-tui) for now.
|
||||||
|
|
||||||
|
You sign in with a username and password. Users have no password until one is
|
||||||
|
set, and a user without one can only use API keys:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# an admin sets someone's first password with their API key
|
||||||
|
curl -X PUT http://localhost:8080/api/users/2/password \
|
||||||
|
-H "Authorization: Bearer $KEY" -H "Content-Type: application/json" \
|
||||||
|
-d '{"password": "<at least 10 characters>"}'
|
||||||
|
```
|
||||||
|
|
||||||
|
After that, users change it themselves under *Account*. Changing your own
|
||||||
|
password requires the current one.
|
||||||
|
|
||||||
|
How a browser stays signed in:
|
||||||
|
|
||||||
|
- A successful login sets an `HttpOnly`, `SameSite=Lax` session cookie. It lasts
|
||||||
|
30 days and slides forward while it is used, so an on-call phone stays signed
|
||||||
|
in.
|
||||||
|
- The cookie is marked `Secure` when `TERDUT_PUBLIC_URL` starts with `https://`,
|
||||||
|
so set it to the HTTPS address. TLS terminates at the gateway and the server
|
||||||
|
itself only ever sees plain HTTP.
|
||||||
|
- Requests authenticated by the cookie are checked for cross-origin use (Go's
|
||||||
|
`http.CrossOriginProtection`). That is the CSRF guard. Bearer-key clients are
|
||||||
|
not affected.
|
||||||
|
- Setting a password signs that user out everywhere else.
|
||||||
|
- Ten failed logins for one username within 15 minutes lock that username for
|
||||||
|
the rest of the window.
|
||||||
|
|
||||||
|
With `TERDUT_PUBLIC_URL` set, tapping a push notification opens the incident in
|
||||||
|
the web UI (`/incidents/{id}`).
|
||||||
|
|
||||||
### Docker
|
### Docker
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -49,6 +94,47 @@ docker run -p 8080:8080 -v $(pwd)/data:/data \
|
|||||||
terdut-server
|
terdut-server
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Kubernetes
|
||||||
|
|
||||||
|
A Helm chart is published from this repository as an OCI artifact, versioned in lockstep
|
||||||
|
with the app — chart `x.y.z` is always app `vx.y.z`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
helm upgrade --install terdut-server oci://git.ryuvia.com/niklas/terdut-server \
|
||||||
|
--version 0.9.2 \
|
||||||
|
--namespace terdut-server --create-namespace \
|
||||||
|
--set networking.hostname=terdut.example.com
|
||||||
|
```
|
||||||
|
|
||||||
|
The chart expects a [Gateway API](https://gateway-api.sigs.k8s.io/) Gateway named `envoy-main` in
|
||||||
|
the `envoy-gateway-system` namespace to already exist — it renders an `HTTPRoute` against it rather
|
||||||
|
than an `Ingress`. TLS is terminated at the gateway, so the server itself never sees a certificate.
|
||||||
|
|
||||||
|
| Value | Default | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| `networking.hostname` | `terdut.example.com` | Hostname the `HTTPRoute` serves |
|
||||||
|
| `networking.listener` | `""` | Gateway listener (`sectionName`) to bind to. Empty attaches to every matching listener, **including plaintext HTTP** — set it to the HTTPS listener's name to serve TLS only |
|
||||||
|
| `networking.servicePort` | `8080` | Port the route forwards to; keep in sync with `service.port` |
|
||||||
|
| `bootstrap.enabled` | `true` | Runs a post-install hook that creates the first user and stores its API key in the `<release>-admin-key` Secret. Already-bootstrapped servers are left alone |
|
||||||
|
| `backupSidecar.enabled` | `true` | Adds an idle `python` sidecar and the [k8up](https://k8up.io/) annotations that dump the database through it |
|
||||||
|
|
||||||
|
The API key travels in an `Authorization: Bearer` header, so set `networking.listener` whenever the
|
||||||
|
hostname is reachable outside a trusted network.
|
||||||
|
|
||||||
|
#### Backups
|
||||||
|
|
||||||
|
The server image is `FROM scratch` — the binary and nothing else — so there is no interpreter to
|
||||||
|
run a database dump in, and the database runs in WAL mode, where a file-level copy of the volume is
|
||||||
|
not crash-consistent. The chart therefore ships an idle `python:*-alpine` sidecar that shares the
|
||||||
|
data volume, and points k8up's `backupcommand` at it with `k8up.io/backupcommand-container`. Without
|
||||||
|
that annotation k8up execs into `.spec.containers[0]` and the dump fails.
|
||||||
|
|
||||||
|
The dump is buffered and sanity-checked before its first byte reaches stdout, because k8up streams
|
||||||
|
stdout straight into Restic: a dump that dies partway is otherwise stored as a silently truncated
|
||||||
|
snapshot that k8up still reports as successful.
|
||||||
|
|
||||||
|
Set `backupSidecar.enabled=false` if you back the volume up some other way.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
@@ -57,6 +143,24 @@ docker run -p 8080:8080 -v $(pwd)/data:/data \
|
|||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `TERDUT_ADDR` | `:8080` | TCP address to listen on |
|
| `TERDUT_ADDR` | `:8080` | TCP address to listen on |
|
||||||
| `TERDUT_DB_PATH` | `terdut.db` | Path to the SQLite database file |
|
| `TERDUT_DB_PATH` | `terdut.db` | Path to the SQLite database file |
|
||||||
|
| `TERDUT_ARCHIVE_AFTER` | `168h` (7d) | How long a resolved alert or incident stays in the default list before being auto-archived |
|
||||||
|
| `TERDUT_STALE_AFTER` | `6h` | How long a firing alert may go without a refreshing webhook before it is treated as resolved — **must exceed your Alertmanager `repeat_interval`** |
|
||||||
|
| `TERDUT_DEADMAN_MATCHERS` | `alertname=Watchdog` | Which alerts are [dead man's switches](#dead-mans-switch). `;` separates matchers, `,` the label conditions within one, `=` is exact equality. Every matcher must name an `alertname` |
|
||||||
|
| `TERDUT_DEADMAN_TIMEOUT` | `15m` | How long a heartbeat may go unheard before its switch is declared dead — **must be shorter than the `repeat_interval` of the route carrying it**. `0` disables dead man's switch handling |
|
||||||
|
| `TERDUT_DEADMAN_SEVERITY` | `critical` | Severity a dead man's switch incident opens at |
|
||||||
|
| `TERDUT_NTFY_URL` | — | ntfy server to publish push notifications to. Empty disables notifications entirely |
|
||||||
|
| `TERDUT_NTFY_TOKEN` | — | Bearer token for an access-controlled ntfy |
|
||||||
|
| `TERDUT_NTFY_FALLBACK_TOPIC` | — | Topic used when nobody is on call |
|
||||||
|
| `TERDUT_PUBLIC_URL` | — | Base URL a phone uses to reach this server: the notification's link into the web UI, its Acknowledge button, and whether the session cookie is `Secure` |
|
||||||
|
| `TERDUT_NOTIFY_REPEAT` | `15m` | How long an incident may sit unacknowledged before it is paged again. `0` notifies once and never repeats |
|
||||||
|
|
||||||
|
Durations use Go syntax (`30m`, `12h`, `168h`). An unparseable value falls back to the default.
|
||||||
|
|
||||||
|
Note that `TERDUT_STALE_AFTER` and `TERDUT_DEADMAN_TIMEOUT` point in opposite directions. Staleness
|
||||||
|
is a generous grace period around a `repeat_interval` you do not control; a dead man's switch is a
|
||||||
|
deadline you set deliberately, and the heartbeat's route is configured to beat faster than it.
|
||||||
|
|
||||||
|
In the Helm chart the two sweeper durations are set via `sweeper.staleAfter` and `sweeper.archiveAfter`, dead man's switches via the `deadman.*` values, and notifications via the `notify.*` values.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -77,26 +181,270 @@ route:
|
|||||||
|
|
||||||
The webhook endpoint requires no authentication.
|
The webhook endpoint requires no authentication.
|
||||||
|
|
||||||
|
If you use the [dead man's switch](#dead-mans-switch) — and the default configuration does — give
|
||||||
|
the heartbeat a route of its own, because the deadline is only as tight as the interval feeding it:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
route:
|
||||||
|
receiver: terdut
|
||||||
|
repeat_interval: 4h
|
||||||
|
routes:
|
||||||
|
- matchers: [ 'alertname = "Watchdog"' ]
|
||||||
|
receiver: terdut
|
||||||
|
group_wait: 0s
|
||||||
|
group_interval: 1m
|
||||||
|
repeat_interval: 1m
|
||||||
|
```
|
||||||
|
|
||||||
|
That delivers a heartbeat every **2 minutes**, not every minute. Alertmanager only reconsiders a
|
||||||
|
group every `group_interval`, and at exactly one elapsed interval `repeat_interval` has not *quite*
|
||||||
|
passed, so the send slips to the next tick — equal values give 2×. Two minutes against the 15 minute
|
||||||
|
default is seven heartbeats per window, which is the point; use `group_interval: 30s` if you want
|
||||||
|
the numbers to mean what they say.
|
||||||
|
|
||||||
|
kube-prometheus-stack users get the `Watchdog` alert (`expr: vector(1)`) for free; it just needs
|
||||||
|
routing to terdut rather than to `null`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Alerts and incidents
|
||||||
|
|
||||||
|
There are two objects, and the difference between them is the whole design.
|
||||||
|
|
||||||
|
**An alert is Alertmanager's record.** It has two states, `firing` and
|
||||||
|
`resolved`, one row per fingerprint, and no human ever writes to it. The API
|
||||||
|
exposes alerts read-only.
|
||||||
|
|
||||||
|
**An incident is the work item.** It goes `triggered → acknowledged → resolved`,
|
||||||
|
carries an assignee, a snooze, notes and a timeline, and is the only thing people
|
||||||
|
act on. Many alerts belong to one incident.
|
||||||
|
|
||||||
|
### Correlation uses Alertmanager's `groupKey`
|
||||||
|
|
||||||
|
Alertmanager has already grouped alerts according to the `group_by` routing tree
|
||||||
|
you configured, and it sends the resulting `groupKey` and `groupLabels` on every
|
||||||
|
webhook. Incidents adopt that answer rather than re-grouping alerts a second
|
||||||
|
time — if you want different correlation, change `group_by` in
|
||||||
|
`alertmanager.yml` and terdut follows.
|
||||||
|
|
||||||
|
At most one incident is open per `groupKey` at a time. Alerts firing in a group
|
||||||
|
that already has an open incident join it. The incident's `severity` is a
|
||||||
|
high-water mark — the highest `severity` label any of its alerts has carried — so
|
||||||
|
an incident that hit `critical` still reads as critical after the critical alert
|
||||||
|
clears.
|
||||||
|
|
||||||
|
### An incident opens only on a new occurrence
|
||||||
|
|
||||||
|
An incident opens when an alert **transitions into firing**: a fingerprint that
|
||||||
|
was never seen, an alert with a newer `startsAt`, or a resolved alert that
|
||||||
|
started again. The unchanged firing notifications Alertmanager re-sends every
|
||||||
|
`repeat_interval` are none of those, and open nothing.
|
||||||
|
|
||||||
|
This is what makes closing an incident by hand mean something. Without the rule,
|
||||||
|
`POST /api/incidents/{id}/resolve` would be undone by the next re-send of an
|
||||||
|
alert that never stopped firing.
|
||||||
|
|
||||||
|
### Leaving the open state
|
||||||
|
|
||||||
|
- **Automatically**, once every alert under the incident has stopped firing —
|
||||||
|
whether by a resolved webhook or by the sweeper's
|
||||||
|
[stale-alert expiry](#stale-alert-expiry). The incident gets
|
||||||
|
`"resolution_source": "alerts"`.
|
||||||
|
- **By hand**, via `POST /api/incidents/{id}/resolve`
|
||||||
|
(`"resolution_source": "manual"`). This is **terminal**: a later occurrence in
|
||||||
|
that group opens a *new* incident rather than reopening this one. If the alert
|
||||||
|
underneath never stops firing, the incident stays closed — that is what
|
||||||
|
resolving by hand asserts.
|
||||||
|
- **On recovery**, for a [dead man's switch](#dead-mans-switch) incident whose
|
||||||
|
heartbeat started arriving again (`"resolution_source": "recovered"`). These
|
||||||
|
incidents have no member alerts, so the automatic cascade above cannot reach
|
||||||
|
them.
|
||||||
|
|
||||||
|
To quieten an incident you expect to come back, snooze it instead
|
||||||
|
(`POST /api/incidents/{id}/snooze`). A snooze hides the incident from the default
|
||||||
|
list without closing it, and expires by simply falling into the past.
|
||||||
|
|
||||||
|
### On-call assignment
|
||||||
|
|
||||||
|
A new incident is assigned to whoever holds today's schedule entry at the moment
|
||||||
|
it opens (`GET /api/schedule/current`). If nobody is scheduled it opens
|
||||||
|
unassigned. Reassign with `POST /api/incidents/{id}/assign`.
|
||||||
|
|
||||||
|
One person holds a given day, so `POST /api/schedule` refuses a date somebody
|
||||||
|
already has: taking a shift off the person expecting to be paged for it should
|
||||||
|
not be something a plain call does by accident. Pass `"replace": true` to take
|
||||||
|
them anyway. Either way the whole request is one transaction — a week where some
|
||||||
|
days are free and some are taken moves as a unit, and a failure leaves the rota
|
||||||
|
exactly as it was rather than with a hole in it.
|
||||||
|
|
||||||
|
### Push notifications
|
||||||
|
|
||||||
|
With `TERDUT_NTFY_URL` set, an incident that opens is pushed to the on-call
|
||||||
|
person's phone through [ntfy](https://ntfy.sh). Set each user's topic with
|
||||||
|
`PUT /api/users/{id}/notify`; a user with no topic falls back to
|
||||||
|
`TERDUT_NTFY_FALLBACK_TOPIC`, as does an incident that opens with nobody on call.
|
||||||
|
If neither yields a topic, nothing is queued.
|
||||||
|
|
||||||
|
Three things get pushed:
|
||||||
|
|
||||||
|
- **triggered** — an incident opened. Priority follows severity (`critical` maps
|
||||||
|
to ntfy's max priority, the one that overrides the phone's quiet settings).
|
||||||
|
- **reminder** — the incident is still `triggered` after `TERDUT_NOTIFY_REPEAT`.
|
||||||
|
Repeats until somebody acts. Acknowledging, snoozing, resolving or archiving
|
||||||
|
all stop it — snooze is the mute button.
|
||||||
|
- **resolved** — every alert under the incident stopped firing. Only sent to
|
||||||
|
whoever was paged in the first place, and only for the automatic cascade:
|
||||||
|
resolving by hand pushes nothing, since the person who did it already knows.
|
||||||
|
|
||||||
|
Notifications carry an **Acknowledge** button that acknowledges the incident
|
||||||
|
without opening anything. It POSTs to `/api/notify/ack/{token}`, an
|
||||||
|
unauthenticated route authorised by the 256-bit token in its path — minted fresh
|
||||||
|
per notification, scoped to one incident and one action, and valid for 24 hours.
|
||||||
|
A real API key is never put in a notification, because the message is stored on
|
||||||
|
the ntfy server and cached on the device.
|
||||||
|
|
||||||
|
The token is **not** consumed by use. Acknowledging is idempotent, so a token
|
||||||
|
stays valid for its full 24 hours and a second tap is a no-op that reports the
|
||||||
|
incident's current state rather than an error — which is what you want when a
|
||||||
|
tap is retried on a flaky mobile connection. What bounds it is scope, not a use
|
||||||
|
count: one incident, one action, one day. Expired tokens are purged by the
|
||||||
|
sweeper.
|
||||||
|
|
||||||
|
Two consequences worth planning for:
|
||||||
|
|
||||||
|
- `/api/notify/ack/{token}` **must stay publicly reachable**, or the button will
|
||||||
|
not work when the responder is off your network.
|
||||||
|
- Notifications sent to the fallback topic carry **no** Acknowledge button. The
|
||||||
|
topic is shared, and a button on it would let any subscriber acknowledge as
|
||||||
|
somebody else.
|
||||||
|
|
||||||
|
Delivery is a queue, not an inline call: the webhook writes a row and a
|
||||||
|
background notifier sends it within 30 seconds, retrying with exponential
|
||||||
|
backoff up to 8 attempts. Nothing about ingestion blocks on ntfy being reachable.
|
||||||
|
|
||||||
|
Every delivery is recorded on the incident's timeline: a `notified` event once
|
||||||
|
ntfy accepts the publish, and a `notify_failed` event when a notification
|
||||||
|
exhausts its retries. Written from the result rather than at enqueue, so the
|
||||||
|
timeline says what actually happened — and a page that never landed is visible
|
||||||
|
instead of looking the same as one that did.
|
||||||
|
|
||||||
|
### Stale alert expiry
|
||||||
|
|
||||||
|
A resolved webhook is the only signal that an alert has stopped firing, so a
|
||||||
|
notification that is dropped, silenced, or lost to a restart would otherwise pin
|
||||||
|
that alert as firing forever. A background sweeper resolves firing alerts that
|
||||||
|
Alertmanager has stopped refreshing, using either signal:
|
||||||
|
|
||||||
|
- the `endsAt` watermark on the last notification has passed, or
|
||||||
|
- no webhook has refreshed the alert within `TERDUT_STALE_AFTER`.
|
||||||
|
|
||||||
|
Alertmanager re-sends firing notifications every `repeat_interval`, which is what
|
||||||
|
keeps a live alert fresh — so `TERDUT_STALE_AFTER` must be comfortably larger
|
||||||
|
than your `repeat_interval` (default 4h), or live alerts will be resolved
|
||||||
|
prematurely. Alerts resolved this way are marked `"resolution_source": "expiry"`
|
||||||
|
to distinguish them from a real Alertmanager resolve (`"alertmanager"`).
|
||||||
|
|
||||||
|
An expiry cascades: once it leaves an incident with nothing firing under it, the
|
||||||
|
incident resolves too, in the same sweep.
|
||||||
|
|
||||||
|
### Dead man's switch
|
||||||
|
|
||||||
|
Everything above assumes alerts arrive. If Prometheus stops evaluating, or
|
||||||
|
Alertmanager cannot reach this server, nothing arrives — and silence looks
|
||||||
|
exactly like everything being fine. A dead man's switch inverts the handling for
|
||||||
|
one designated alert so that silence is the signal:
|
||||||
|
|
||||||
|
- **receiving** it opens no incident, and
|
||||||
|
- the **absence** of it does.
|
||||||
|
|
||||||
|
kube-prometheus-stack already ships the alert for this. `Watchdog` is
|
||||||
|
`expr: vector(1)`, so it fires permanently and is re-sent forever; it is worth
|
||||||
|
nothing unless something downstream notices it stop. That is what
|
||||||
|
`TERDUT_DEADMAN_MATCHERS` defaults to.
|
||||||
|
|
||||||
|
A matcher is a set of exact label conditions, one of which must be the
|
||||||
|
`alertname`:
|
||||||
|
|
||||||
|
```
|
||||||
|
TERDUT_DEADMAN_MATCHERS="alertname=Watchdog,cluster=prod; alertname=EdgeHeartbeat"
|
||||||
|
```
|
||||||
|
|
||||||
|
**The unit of monitoring is the fingerprint, not the alert name.** Two clusters
|
||||||
|
sending the same `Watchdog` are two independent switches, so a healthy one can
|
||||||
|
never mask a dead one.
|
||||||
|
|
||||||
|
#### The lifecycle
|
||||||
|
|
||||||
|
A switch is **dormant** until its first heartbeat arrives. A configured matcher
|
||||||
|
that has never been heard from opens nothing, so a fresh deploy or a restored
|
||||||
|
database does not page. It also means a matcher that never matches anything is
|
||||||
|
silently inert — check the startup log line, which lists the matchers that
|
||||||
|
survived parsing.
|
||||||
|
|
||||||
|
Once armed, the sweeper declares it **dead** when either the heartbeat has not
|
||||||
|
been refreshed within `TERDUT_DEADMAN_TIMEOUT`, or Alertmanager explicitly
|
||||||
|
resolved it — the sender saying the heartbeat stopped needs no further waiting.
|
||||||
|
That opens an incident at `TERDUT_DEADMAN_SEVERITY`, assigned and paged like any
|
||||||
|
other, and marks the heartbeat alert `"resolution_source": "deadman"` so the
|
||||||
|
alert list stops claiming a dead switch is firing.
|
||||||
|
|
||||||
|
It **recovers** when the heartbeat starts arriving again: the incident resolves
|
||||||
|
with `"resolution_source": "recovered"` and the all-clear goes to whoever was
|
||||||
|
paged.
|
||||||
|
|
||||||
|
Resolving the incident by hand sticks, the same way it does for an alert-backed
|
||||||
|
one. While the switch stays silent nothing new opens — so a decommissioned
|
||||||
|
source is a one-time page rather than a nag. The switch **re-arms** on the next
|
||||||
|
heartbeat: come back and die again, and that is a new incident.
|
||||||
|
|
||||||
|
#### Two things to know
|
||||||
|
|
||||||
|
`TERDUT_DEADMAN_TIMEOUT` must be **shorter** than the `repeat_interval` of the
|
||||||
|
route carrying the heartbeat, which is the exact opposite of
|
||||||
|
`TERDUT_STALE_AFTER`. Inheriting a default `repeat_interval` of 4h gives you a
|
||||||
|
switch that takes four hours to notice anything, so give the heartbeat
|
||||||
|
[its own route](#alertmanager-configuration). Matched alerts are exempt from
|
||||||
|
stale-alert expiry — a heartbeat answers to its own timeout and nothing else.
|
||||||
|
|
||||||
|
A dead man's switch incident has **no member alerts**:
|
||||||
|
`GET /api/incidents/{id}/alerts` returns an empty list. There is no alert
|
||||||
|
describing the problem, because the problem is that no alert arrived. What
|
||||||
|
happened is on the timeline instead, as a `deadman_silent` event carrying the age
|
||||||
|
of the last heartbeat, and the heartbeat's labels are on the incident's
|
||||||
|
`group_labels`.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## API reference
|
## API reference
|
||||||
|
|
||||||
### Authentication
|
### Authentication
|
||||||
|
|
||||||
All endpoints except `/api/bootstrap` and `/api/alertmanager/webhook` require:
|
All endpoints except `/api/bootstrap`, `/api/alertmanager/webhook`,
|
||||||
|
`/api/notify/ack/{token}`, `/api/login` and `/api/logout` require either an API key:
|
||||||
|
|
||||||
```
|
```
|
||||||
Authorization: Bearer <api-key>
|
Authorization: Bearer <api-key>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
or the web UI's session cookie. A request that carries an `Authorization` header
|
||||||
|
is judged on that header alone.
|
||||||
|
|
||||||
|
| Method | Path | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| `POST` | `/api/login` | `{"username","password"}` → sets the session cookie, returns `{user, has_password}`. `429` after too many failures |
|
||||||
|
| `POST` | `/api/logout` | Ends the session and clears the cookie |
|
||||||
|
| `GET` | `/api/me` | The caller: `{user, has_password}` |
|
||||||
|
|
||||||
### Users
|
### Users
|
||||||
|
|
||||||
| Method | Path | Description |
|
| Method | Path | Description |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `POST` | `/api/bootstrap` | Create first user + API key (only works on empty DB) |
|
| `POST` | `/api/bootstrap` | Create first user + API key `{"username","email","password"?}` (only works on empty DB) |
|
||||||
| `GET` | `/api/users` | List users |
|
| `GET` | `/api/users` | List users |
|
||||||
| `POST` | `/api/users` | Create user `{"username","email"}` |
|
| `POST` | `/api/users` | Create user `{"username","email"}` |
|
||||||
| `DELETE` | `/api/users/{id}` | Delete user (cascades to keys) |
|
| `DELETE` | `/api/users/{id}` | Delete user (cascades to keys) |
|
||||||
|
| `PUT` | `/api/users/{id}/notify` | Set push notification target `{"ntfy_topic"}` — empty string clears it |
|
||||||
|
| `PUT` | `/api/users/{id}/password` | Set web UI password `{"password","current_password"}`. `current_password` is required only when changing your own existing password. Ends the user's other sessions |
|
||||||
| `POST` | `/api/users/{id}/api-keys` | Issue API key `{"name"}` — key shown once |
|
| `POST` | `/api/users/{id}/api-keys` | Issue API key `{"name"}` — key shown once |
|
||||||
| `DELETE` | `/api/users/{id}/api-keys/{keyID}` | Revoke API key |
|
| `DELETE` | `/api/users/{id}/api-keys/{keyID}` | Revoke API key |
|
||||||
|
|
||||||
@@ -106,38 +454,259 @@ Authorization: Bearer <api-key>
|
|||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `POST` | `/api/alertmanager/webhook` | Alertmanager v4 webhook receiver (no auth) |
|
| `POST` | `/api/alertmanager/webhook` | Alertmanager v4 webhook receiver (no auth) |
|
||||||
|
|
||||||
### Alerts
|
### Notifications
|
||||||
|
|
||||||
| Method | Path | Description |
|
| Method | Path | Description |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `GET` | `/api/alerts` | List alerts. Filters: `?status=firing\|resolved`, `?name=`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?limit=` (default 50, max 500) |
|
| `POST` | `/api/notify/ack/{token}` | Acknowledge an incident from a push notification's Acknowledge button. No auth: the token in the path is the credential — one incident, one action, 24 hours, idempotent. Must stay publicly reachable |
|
||||||
|
|
||||||
|
### Incidents
|
||||||
|
|
||||||
|
| Method | Path | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| `GET` | `/api/incidents` | List incidents. Filters: `?status=triggered\|acknowledged\|resolved`, `?severity=`, `?assigned_to=<user id>`, `?archived=true`, `?snoozed=true`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?sort=severity`, `?limit=` (default 50, max 500) |
|
||||||
|
| `GET` | `/api/incidents/{id}` | Get single incident, with its alerts inline |
|
||||||
|
| `GET` | `/api/incidents/{id}/alerts` | Alerts under this incident |
|
||||||
|
| `GET` | `/api/incidents/{id}/timeline` | Full event history, chronological |
|
||||||
|
| `POST` | `/api/incidents/{id}/acknowledge` | Acknowledge (stamps authed user + time) |
|
||||||
|
| `DELETE` | `/api/incidents/{id}/acknowledge` | Clear acknowledgement, back to `triggered` |
|
||||||
|
| `POST` | `/api/incidents/{id}/resolve` | Close by hand — **terminal**, see above |
|
||||||
|
| `POST` | `/api/incidents/{id}/assign` | Reassign `{"user_id"}` |
|
||||||
|
| `POST` | `/api/incidents/{id}/snooze` | Hide until `{"until": RFC3339}` or `{"duration": "2h"}` |
|
||||||
|
| `DELETE` | `/api/incidents/{id}/snooze` | Un-snooze |
|
||||||
|
| `POST` | `/api/incidents/{id}/archive` | Archive (hides from the default list) |
|
||||||
|
| `DELETE` | `/api/incidents/{id}/archive` | Un-archive |
|
||||||
|
| `POST` | `/api/incidents/{id}/notes` | Add a note `{"content"}` |
|
||||||
|
| `DELETE` | `/api/incidents/{id}/notes/{eventID}` | Delete own note |
|
||||||
|
|
||||||
|
With no `?status=` filter, `GET /api/incidents` returns **open** incidents only —
|
||||||
|
the queue an on-call person wants. Currently snoozed and archived incidents are
|
||||||
|
excluded unless asked for. Actions that only make sense on an open incident
|
||||||
|
return `409` once it is resolved.
|
||||||
|
|
||||||
|
Notes are ordinary timeline events of type `note`; only they are deletable, and
|
||||||
|
only by their author. The rest of the timeline is a record of what happened.
|
||||||
|
|
||||||
|
#### The incident object
|
||||||
|
|
||||||
|
| Field | Type | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| `id` | integer | Server-assigned |
|
||||||
|
| `group_key` | string | Alertmanager's `groupKey` — opaque, treat as an identifier |
|
||||||
|
| `title` | string | Rendered from `groupLabels` |
|
||||||
|
| `group_labels` | object | String→string, as sent by Alertmanager |
|
||||||
|
| `status` | string | `"triggered"`, `"acknowledged"` or `"resolved"` |
|
||||||
|
| `severity` | string | *optional* — high-water mark across the incident's alerts; never lowered |
|
||||||
|
| `triggered_at` | timestamp | When the incident opened |
|
||||||
|
| `acknowledged_by_id` / `acknowledged_by` / `acknowledged_at` | | *optional* — user id, username, time |
|
||||||
|
| `assigned_to_id` / `assigned_to` | | *optional* — user id, username |
|
||||||
|
| `snoozed_until` | timestamp | *optional* — a value in the past reads as not snoozed |
|
||||||
|
| `resolved_at` | timestamp | *optional* |
|
||||||
|
| `resolution_source` | string | *optional* — `"alerts"`, `"manual"` or `"recovered"` |
|
||||||
|
| `archived_at` | timestamp | *optional* |
|
||||||
|
| `alerts` | array | Only on `GET /api/incidents/{id}` |
|
||||||
|
|
||||||
|
Treat `resolution_source` as an open set, as with the alert field of the same
|
||||||
|
name: degrade unknown values to "resolved, reason unknown".
|
||||||
|
|
||||||
|
#### The timeline event object
|
||||||
|
|
||||||
|
| Field | Type | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| `id` | integer | |
|
||||||
|
| `incident_id` | integer | |
|
||||||
|
| `type` | string | See below — treat as an open set |
|
||||||
|
| `user_id` / `username` | | *optional* — absent when the server acted rather than a person |
|
||||||
|
| `alert_id` | integer | *optional* — the alert an `alert_added` / `alert_resolved` event refers to |
|
||||||
|
| `detail` | string | *optional* — the note body, the snooze deadline, etc. |
|
||||||
|
| `created_at` | timestamp | |
|
||||||
|
|
||||||
|
Types written today: `triggered`, `alert_added`, `alert_resolved`,
|
||||||
|
`acknowledged`, `unacknowledged`, `assigned`, `snoozed`, `unsnoozed`, `resolved`,
|
||||||
|
`note`, `notified`, `notify_failed`, `deadman_silent`. On an `assigned` event
|
||||||
|
`user_id` is the **assignee**, not the actor. New types may be added; render
|
||||||
|
unknown ones generically rather than dropping them.
|
||||||
|
|
||||||
|
On `notified` and `notify_failed`, `detail` carries the notification kind
|
||||||
|
(`triggered` | `reminder` | `resolved`), and on a failure the reason after it.
|
||||||
|
`user_id` is who was paged — absent means the page went to the shared fallback
|
||||||
|
topic and so belongs to nobody. The topic itself is never written to the
|
||||||
|
timeline: it is a shared secret with the ntfy server, and every API key can read
|
||||||
|
this.
|
||||||
|
|
||||||
|
### Alerts
|
||||||
|
|
||||||
|
Alerts are read-only. Everything a person does happens on the incident.
|
||||||
|
|
||||||
|
| Method | Path | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| `GET` | `/api/alerts` | List alerts. Filters: `?status=firing\|resolved`, `?name=`, `?incident_id=`, `?archived=true`, `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD`, `?limit=` (default 50, max 500) |
|
||||||
| `GET` | `/api/alerts/{id}` | Get single alert |
|
| `GET` | `/api/alerts/{id}` | Get single alert |
|
||||||
| `POST` | `/api/alerts/{id}/acknowledge` | Acknowledge alert (stamps authed user + time) |
|
|
||||||
| `DELETE` | `/api/alerts/{id}/acknowledge` | Clear acknowledgement |
|
Archived alerts are hidden from `GET /api/alerts` unless `?archived=true` is
|
||||||
| `GET` | `/api/alerts/{id}/comments` | List comments (chronological) |
|
passed; alert archiving is automatic housekeeping by the sweeper, not a user
|
||||||
| `POST` | `/api/alerts/{id}/comments` | Add comment `{"content"}` |
|
action. Resolved alerts carry `resolution_source`: `"alertmanager"` for a real
|
||||||
| `DELETE` | `/api/alerts/{id}/comments/{commentID}` | Delete own comment |
|
resolved webhook, `"expiry"` when the sweeper inferred it (see
|
||||||
|
[Stale alert expiry](#stale-alert-expiry)), `"deadman"` for a heartbeat declared
|
||||||
|
dead (see [Dead man's switch](#dead-mans-switch)).
|
||||||
|
|
||||||
|
#### The alert object
|
||||||
|
|
||||||
|
Returned by `GET /api/alerts` (as an array) and `GET /api/alerts/{id}`.
|
||||||
|
Timestamps are RFC 3339 in UTC. Fields marked *optional* are omitted entirely
|
||||||
|
when unset, so clients must treat them as nullable.
|
||||||
|
|
||||||
|
| Field | Type | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| `id` | integer | Server-assigned; stable for the life of the row |
|
||||||
|
| `fingerprint` | string | Alertmanager's fingerprint — the upsert key |
|
||||||
|
| `name` | string | From the `alertname` label |
|
||||||
|
| `status` | string | `"firing"` or `"resolved"` |
|
||||||
|
| `labels` | object | String→string, as sent by Alertmanager |
|
||||||
|
| `annotations` | object | String→string, as sent by Alertmanager |
|
||||||
|
| `starts_at` | timestamp | When the alert instance began, **per Prometheus** |
|
||||||
|
| `ends_at` | timestamp | *optional* — absent while no end is known |
|
||||||
|
| `generator_url` | string | Link back to the originating Prometheus |
|
||||||
|
| `received_at` | timestamp | When the server last accepted a webhook for this alert — see below |
|
||||||
|
| `incident_id` | integer | *optional* — the most recent incident this alert belongs to |
|
||||||
|
| `resolution_source` | string | *optional* — `"alertmanager"`, `"expiry"` or `"deadman"` |
|
||||||
|
| `archived_at` | timestamp | *optional* — set while archived |
|
||||||
|
|
||||||
|
##### `received_at` is a liveness heartbeat
|
||||||
|
|
||||||
|
`starts_at` comes from Prometheus and **never changes** for the lifetime of an
|
||||||
|
alert instance. It says when the problem began, not whether it is still
|
||||||
|
happening — an alert that started twelve days ago looks identical whether
|
||||||
|
Alertmanager refreshed it a minute ago or went silent a week ago.
|
||||||
|
|
||||||
|
`received_at` is the field that answers "is this still live". It is set to the
|
||||||
|
server's clock on **every accepted webhook** for that fingerprint, including the
|
||||||
|
unchanged firing notifications Alertmanager re-sends every `repeat_interval`.
|
||||||
|
Clients may rely on this:
|
||||||
|
|
||||||
|
- **A firing alert whose `received_at` is advancing is still being refreshed.**
|
||||||
|
Stale-dating it against `repeat_interval` is a valid liveness check, and it is
|
||||||
|
what the built-in sweeper does (see
|
||||||
|
[Stale alert expiry](#stale-alert-expiry)).
|
||||||
|
- **`received_at` tracks accepted payloads, not delivery attempts.** A retry
|
||||||
|
that describes an older instance than the stored one is discarded, and a
|
||||||
|
discarded payload does not move `received_at`.
|
||||||
|
- **It stops advancing once the alert resolves,** because Alertmanager stops
|
||||||
|
re-sending. On an alert resolved by the sweeper
|
||||||
|
(`"resolution_source": "expiry"`) it therefore marks the last time
|
||||||
|
Alertmanager was actually heard from, which is earlier than `ends_at`.
|
||||||
|
|
||||||
|
`GET /api/alerts` is ordered by `received_at` descending — most recently
|
||||||
|
refreshed first — and the `?from=` / `?to=` filters on both the alert and stats
|
||||||
|
endpoints select on `received_at`, not `starts_at`.
|
||||||
|
|
||||||
|
##### `resolution_source` says how much to trust `ends_at`
|
||||||
|
|
||||||
|
An alert can leave the firing state two ways, and `resolution_source` records
|
||||||
|
which happened. Clients may rely on this:
|
||||||
|
|
||||||
|
- **Absent while firing.** It is set only on resolve, and a re-fire under the
|
||||||
|
same fingerprint clears it again, so its presence always agrees with
|
||||||
|
`"status": "resolved"`.
|
||||||
|
- **`"alertmanager"` — a real resolved webhook arrived.** `ends_at` is the end
|
||||||
|
time Alertmanager reported. It is an observed value and can be displayed as
|
||||||
|
fact.
|
||||||
|
- **`"expiry"` — the sweeper inferred the resolve** because Alertmanager stopped
|
||||||
|
refreshing the alert (see [Stale alert expiry](#stale-alert-expiry)). Nothing
|
||||||
|
ever reported an end, so **`ends_at` is approximate**: it is either the stale
|
||||||
|
`endsAt` watermark from the last notification, or — when that notification
|
||||||
|
carried none — the time the sweep ran, which lags the last real contact by up
|
||||||
|
to `TERDUT_STALE_AFTER` plus a sweep interval. Treat it as "no later than",
|
||||||
|
not as when the problem stopped.
|
||||||
|
|
||||||
|
On these alerts `received_at` is the more truthful signal: it marks the last
|
||||||
|
time Alertmanager was actually heard from. Surfacing the distinction is
|
||||||
|
worthwhile, since `"expiry"` can also mean the alert is still firing and the
|
||||||
|
notification path broke.
|
||||||
|
|
||||||
|
- **`"deadman"` — a heartbeat was declared dead** (see
|
||||||
|
[Dead man's switch](#dead-mans-switch)). Like `"expiry"`, an inference from
|
||||||
|
silence rather than an observed end, so `ends_at` is approximate — but a much
|
||||||
|
tighter one, bounded by `TERDUT_DEADMAN_TIMEOUT`. It is also the one resolution
|
||||||
|
a re-fire under the same `starts_at` can undo, since the switch coming back is
|
||||||
|
exactly the evidence that the inference was wrong.
|
||||||
|
|
||||||
|
Treat the value as an open set and tolerate ones you do not recognise — new
|
||||||
|
sources may be added, and unknown values should degrade to "resolved, reason
|
||||||
|
unknown" rather than being rejected.
|
||||||
|
|
||||||
### On-call schedule
|
### On-call schedule
|
||||||
|
|
||||||
| Method | Path | Description |
|
| Method | Path | Description |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `POST` | `/api/schedule` | Assign user to dates `{"user_id", "dates":["YYYY-MM-DD",...]}` — all-or-nothing |
|
| `POST` | `/api/schedule` | Assign user to dates `{"user_id", "dates":["YYYY-MM-DD",...], "replace"}` — all-or-nothing |
|
||||||
| `GET` | `/api/schedule` | List entries. Filters: `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD` |
|
| `GET` | `/api/schedule` | List entries. Filters: `?from=YYYY-MM-DD`, `?to=YYYY-MM-DD` |
|
||||||
| `GET` | `/api/schedule/current` | Today's on-call user (UTC), 404 if none |
|
| `GET` | `/api/schedule/current` | Today's on-call user (UTC), 404 if none |
|
||||||
| `DELETE` | `/api/schedule/{id}` | Remove schedule entry |
|
| `DELETE` | `/api/schedule/{id}` | Remove schedule entry |
|
||||||
|
|
||||||
### Statistics
|
### Statistics
|
||||||
|
|
||||||
All stat endpoints accept optional `?from=YYYY-MM-DD` and `?to=YYYY-MM-DD` to filter by `received_at`.
|
All stat endpoints accept optional `?from=YYYY-MM-DD` and `?to=YYYY-MM-DD`, and exclude archived rows to match the default list views. Alert stats filter on `received_at`; incident stats filter on `triggered_at`.
|
||||||
|
|
||||||
| Method | Path | Description |
|
| Method | Path | Description |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
|
| `GET` | `/api/stats/incidents` | `{total, triggered, acknowledged, resolved, mtta_seconds, mttr_seconds}` |
|
||||||
| `GET` | `/api/stats/alerts` | `{total, firing, resolved}` counts |
|
| `GET` | `/api/stats/alerts` | `{total, firing, resolved}` counts |
|
||||||
| `GET` | `/api/stats/alerts/top` | Most frequent alert names. `?limit=` (default 10, max 100) |
|
| `GET` | `/api/stats/alerts/top` | Most frequent alert names. `?limit=` (default 10, max 100) |
|
||||||
| `GET` | `/api/stats/alerts/by-hour` | Count per hour-of-day (UTC), all 24 slots returned |
|
| `GET` | `/api/stats/alerts/by-hour` | Count per hour-of-day (UTC), all 24 slots returned |
|
||||||
| `GET` | `/api/stats/alerts/by-day` | Count per day-of-week, all 7 slots with names returned |
|
| `GET` | `/api/stats/alerts/by-day` | Count per day-of-week, all 7 slots with names returned |
|
||||||
|
|
||||||
|
`mtta_seconds` (time to acknowledge) and `mttr_seconds` (time to resolve) are
|
||||||
|
averages over incidents that have actually been acknowledged or resolved, and are
|
||||||
|
**null** until there are any — null means "no data", not zero.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Upgrading to incidents
|
||||||
|
|
||||||
|
The incidents release moves the workflow off alerts, which is a **breaking API
|
||||||
|
change**. These endpoints are gone:
|
||||||
|
|
||||||
|
| Removed | Replacement |
|
||||||
|
|---|---|
|
||||||
|
| `POST`/`DELETE` `/api/alerts/{id}/acknowledge` | `POST`/`DELETE` `/api/incidents/{id}/acknowledge` |
|
||||||
|
| `POST`/`DELETE` `/api/alerts/{id}/archive` | `POST`/`DELETE` `/api/incidents/{id}/archive` (alert archiving is now sweeper-only) |
|
||||||
|
| `GET`/`POST` `/api/alerts/{id}/comments` | `GET /api/incidents/{id}/timeline`, `POST /api/incidents/{id}/notes` |
|
||||||
|
| `DELETE /api/alerts/{id}/comments/{commentID}` | `DELETE /api/incidents/{id}/notes/{eventID}` |
|
||||||
|
|
||||||
|
The alert object also drops `acknowledged_by_id`, `acknowledged_by` and
|
||||||
|
`acknowledged_at`, and gains `incident_id`.
|
||||||
|
|
||||||
|
Migration `008_incidents.sql` runs automatically on start and preserves existing
|
||||||
|
data: every alert gets a backfilled incident carrying its acknowledgement, and
|
||||||
|
comments become timeline notes. Backfilled incidents have a `group_key` of
|
||||||
|
`backfill:<fingerprint>` — there is no historical `groupKey` to correlate on, so
|
||||||
|
they are one-per-alert rather than grouped.
|
||||||
|
|
||||||
|
Nothing about the two documented alert contracts changes: `received_at` is still
|
||||||
|
advanced on every accepted webhook, and `resolution_source` still means what it
|
||||||
|
did.
|
||||||
|
|
||||||
|
## Upgrading to dead man's switches
|
||||||
|
|
||||||
|
Dead man's switch handling is **on by default**, watching `alertname=Watchdog`
|
||||||
|
with a 15 minute timeout. If you already route `Watchdog` to this server, the
|
||||||
|
behaviour of that alert changes on upgrade, in both directions:
|
||||||
|
|
||||||
|
- it stops opening incidents when it arrives, and
|
||||||
|
- it starts opening one when it stops arriving.
|
||||||
|
|
||||||
|
**Check your `repeat_interval` before upgrading.** The switch pages whenever a
|
||||||
|
heartbeat has not been refreshed within `TERDUT_DEADMAN_TIMEOUT`, so a `Watchdog`
|
||||||
|
route inheriting a 4h or 12h `repeat_interval` will page constantly against the
|
||||||
|
15 minute default. Either give the heartbeat
|
||||||
|
[its own fast route](#alertmanager-configuration) — the point of the feature — or
|
||||||
|
set `TERDUT_DEADMAN_TIMEOUT` above your current `repeat_interval` until you have.
|
||||||
|
`TERDUT_DEADMAN_TIMEOUT=0` turns the whole thing off.
|
||||||
|
|
||||||
|
There is no migration and no schema change. An existing open incident from a
|
||||||
|
`Watchdog` that arrived under the old behaviour is unaffected; resolve it by hand.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
@@ -147,3 +716,68 @@ go test ./... # run all tests
|
|||||||
go build ./... # compile all packages
|
go build ./... # compile all packages
|
||||||
go run ./cmd/terdut # run locally
|
go run ./cmd/terdut # run locally
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`make fmt lint test helm-lint` is the gate. It mirrors `.gitea/workflows/ci.yaml` step for
|
||||||
|
step, so a green run here means a green pipeline — with one deliberate exception: `make test`
|
||||||
|
adds `-race`, which CI does not. The sweeper, the notifier goroutine and the dead man's switch
|
||||||
|
sweep all touch the same single database connection, and a race between them would surface as
|
||||||
|
a flaky incident in production rather than as a red build.
|
||||||
|
|
||||||
|
The web UI lives in `internal/web/static/` as plain HTML, CSS and ES modules,
|
||||||
|
embedded into the binary with `go:embed`. It has no build step and no npm, so
|
||||||
|
editing a file and restarting the server is the whole loop.
|
||||||
|
|
||||||
|
## Releasing
|
||||||
|
|
||||||
|
```
|
||||||
|
push or PR → ci.yaml gofmt, go vet, go test -race
|
||||||
|
govulncheck, gitleaks
|
||||||
|
helm lint + render
|
||||||
|
push tag vX.Y.Z → release.yaml the same gate, then publish:
|
||||||
|
git.ryuvia.com/niklas/terdut-server:vX.Y.Z
|
||||||
|
oci://git.ryuvia.com/niklas/terdut-server X.Y.Z
|
||||||
|
then trivy-scan the pushed image
|
||||||
|
PR to Ryuvia/charts → bump the wrapper chart to X.Y.Z; on merge
|
||||||
|
Flux reconciles and the release rolls out
|
||||||
|
```
|
||||||
|
|
||||||
|
Both artifacts go to the **personal** Gitea namespace rather than `ryuvia`, because Gitea
|
||||||
|
scopes package visibility to the owner with no per-package override — so `ryuvia/*` is private
|
||||||
|
because the org is. Publishing to `niklas` keeps them anonymously pullable, which is why no
|
||||||
|
pull secret is needed in the cluster. Same reasoning, and the same choice, as riksdata and
|
||||||
|
rd-web.
|
||||||
|
|
||||||
|
Saying **"Release"** runs all three rows: the `release` skill commits, pushes, tags, waits for
|
||||||
|
the pipeline, and opens the `Ryuvia/charts` PR, stopping before the merge. See
|
||||||
|
`~/.claude/skills/release/`, or `.release.conf` here for this repo's part of it.
|
||||||
|
|
||||||
|
The chart is published **only** from the tag, by the `chart` job. There used to be a second
|
||||||
|
publisher on every `charts/**` push to main, and the two raced for the same chart version with
|
||||||
|
different answers — chart 0.9.0 went out reading `appVersion: "latest"` that way. One
|
||||||
|
publisher, triggered by the tag (`766f439`). The cost is that a chart-only change has no
|
||||||
|
version of its own and rides the next app tag.
|
||||||
|
|
||||||
|
Both workflows are thin drivers over the Makefile: `ci.yaml` runs `make fmt lint test` and
|
||||||
|
`make helm-lint`, `release.yaml` adds `make binaries`, `make push`, `make helm-package` and
|
||||||
|
`make helm-push`. That is deliberate — it is what makes a green local gate and a green
|
||||||
|
pipeline the same code rather than two descriptions of it, and it is how riksdata and rd-web
|
||||||
|
have always worked.
|
||||||
|
|
||||||
|
`make push` builds and pushes in one step, unlike those two, because the image is
|
||||||
|
`linux/amd64,linux/arm64` and buildx cannot load a multi-platform result into the local image
|
||||||
|
store. `make build` stays single-platform and local-only. Both refuse `VERSION=dev`:
|
||||||
|
publishing is one command, so it is also one command to run by accident. Publishing happens
|
||||||
|
by pushing a tag.
|
||||||
|
|
||||||
|
Two things the release process needs to know about this repo:
|
||||||
|
|
||||||
|
- **The image scan runs after publishing**, like riksdata's and rd-web's: trivy cannot read
|
||||||
|
a locally built image on this runner, so it pulls the pushed one. A red `scan-image` means
|
||||||
|
do not bump the wrapper chart to that version — it cannot unpublish anything. The image is
|
||||||
|
`FROM scratch`, so trivy sees exactly one target, the Go binary and its module graph.
|
||||||
|
- **The wrapper chart's `values.yaml` has two `tag:` lines** — the app image and the python
|
||||||
|
backup sidecar — so `chart-bump` is given `--image` to say which one moves.
|
||||||
|
|
||||||
|
The wrapper chart must have **its own `version:` bumped in the same commit**. Flux reconciles
|
||||||
|
with `reconcileStrategy: ChartVersion`, so a chart whose version did not change produces no
|
||||||
|
new artifact and the change is never deployed — with no error anywhere.
|
||||||
|
|||||||
@@ -2,5 +2,18 @@ apiVersion: v2
|
|||||||
name: terdut-server
|
name: terdut-server
|
||||||
description: A Helm chart for Terminal Duty — on-call alert management server
|
description: A Helm chart for Terminal Duty — on-call alert management server
|
||||||
type: application
|
type: application
|
||||||
version: 0.2.0
|
# These two are placeholders for a local `helm install ./charts/terdut-server`, not the
|
||||||
appVersion: "latest"
|
# released values. .gitea/workflows/release.yaml rewrites both from the git tag when it
|
||||||
|
# publishes, so the chart version always equals the app version.
|
||||||
|
#
|
||||||
|
# They are kept in step with the tag anyway. Being read is the only thing these two lines
|
||||||
|
# do -- `helm package --version --app-version` sets the published values from the tag and
|
||||||
|
# never consults these -- and a tree heading for a numbered release that states an older
|
||||||
|
# number tells its reader something false. They said 0.9.0 and "latest" until 2026-09-01,
|
||||||
|
# through two releases.
|
||||||
|
#
|
||||||
|
# appVersion and image.tag in values.yaml no longer agree, and that is not an oversight:
|
||||||
|
# image.tag stays "latest", which is what a local install actually pulls. appVersion is
|
||||||
|
# metadata and drives nothing.
|
||||||
|
version: 0.10.0
|
||||||
|
appVersion: "v0.10.0"
|
||||||
|
|||||||
@@ -23,13 +23,23 @@ spec:
|
|||||||
serviceAccountName: {{ include "terdut-server.fullname" . }}-bootstrap
|
serviceAccountName: {{ include "terdut-server.fullname" . }}-bootstrap
|
||||||
containers:
|
containers:
|
||||||
- name: bootstrap
|
- name: bootstrap
|
||||||
image: alpine:3
|
# alpine/curl, not alpine:3 + `apk add curl`. Installing the binary at run time
|
||||||
|
# writes it into the container's writable upper layer, which is exactly the
|
||||||
|
# signature Falco's `Drop and execute new binary in container` (MITRE TA0003)
|
||||||
|
# exists to catch -- this hook emitted two Critical events on every single
|
||||||
|
# upgrade. See Ryuvia/charts#100. It also made `helm upgrade` depend on the
|
||||||
|
# Alpine CDN answering, since this runs as a post-upgrade hook and a failed
|
||||||
|
# hook fails the release.
|
||||||
|
#
|
||||||
|
# Still a full Alpine underneath, so sh, cat, sleep, grep, cut, head and tail
|
||||||
|
# are all present (verified in-cluster 2026-09-04). The image declares
|
||||||
|
# ENTRYPOINT ["/entrypoint.sh"], which `command:` below overrides -- do not
|
||||||
|
# change `command:` to `args:`.
|
||||||
|
image: alpine/curl:8.21.0@sha256:a1c44bab54d88e18ea9a6a4ecefab7f2d230b968567b78960fcaff8d51b7f067
|
||||||
command:
|
command:
|
||||||
- /bin/sh
|
- /bin/sh
|
||||||
- -c
|
- -c
|
||||||
- |
|
- |
|
||||||
apk add --no-cache curl > /dev/null 2>&1
|
|
||||||
|
|
||||||
SERVICE_URL="http://{{ include "terdut-server.fullname" . }}:{{ .Values.service.port }}"
|
SERVICE_URL="http://{{ include "terdut-server.fullname" . }}:{{ .Values.service.port }}"
|
||||||
SECRET_NAME="{{ include "terdut-server.bootstrapSecretName" . }}"
|
SECRET_NAME="{{ include "terdut-server.bootstrapSecretName" . }}"
|
||||||
K8S_API="https://kubernetes.default.svc"
|
K8S_API="https://kubernetes.default.svc"
|
||||||
|
|||||||
@@ -10,10 +10,50 @@ spec:
|
|||||||
selector:
|
selector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
{{- include "terdut-server.selectorLabels" . | nindent 6 }}
|
{{- include "terdut-server.selectorLabels" . | nindent 6 }}
|
||||||
|
# The data PVC is ReadWriteOnce, so a RollingUpdate deadlocks: the new pod
|
||||||
|
# cannot attach the volume until the old one releases it, and the old one is
|
||||||
|
# not torn down until the new one is ready.
|
||||||
|
strategy:
|
||||||
|
type: Recreate
|
||||||
template:
|
template:
|
||||||
metadata:
|
metadata:
|
||||||
labels:
|
labels:
|
||||||
{{- include "terdut-server.selectorLabels" . | nindent 8 }}
|
{{- include "terdut-server.selectorLabels" . | nindent 8 }}
|
||||||
|
{{- if .Values.backupSidecar.enabled }}
|
||||||
|
annotations:
|
||||||
|
# Dumps the whole database: incidents, alerts, users, API key hashes,
|
||||||
|
# the schedule and the notification outbox.
|
||||||
|
#
|
||||||
|
# Runs in the `backup` sidecar, NOT in the app container: the server
|
||||||
|
# image is FROM scratch and has no interpreter at all. k8up execs into
|
||||||
|
# .spec.containers[0] unless told otherwise, hence the explicit
|
||||||
|
# k8up.io/backupcommand-container.
|
||||||
|
#
|
||||||
|
# Buffered and sanity-checked before the first byte reaches stdout: k8up
|
||||||
|
# streams stdout straight into restic, so a dump that dies partway is
|
||||||
|
# stored as a silently-truncated snapshot that k8up still reports as
|
||||||
|
# Succeeded. The check counts users rather than incidents -- incidents
|
||||||
|
# are swept and archived, so an empty incidents table is a legitimate
|
||||||
|
# state, whereas a database with no users never is.
|
||||||
|
#
|
||||||
|
# The connection is read-only but the mount is not: the database runs in
|
||||||
|
# WAL mode, and opening it mode=ro still needs write access to the -shm
|
||||||
|
# wal-index.
|
||||||
|
#
|
||||||
|
# chr(10), not '\n': k8up parses this annotation with go-shellquote.
|
||||||
|
k8up.io/backupcommand-container: backup
|
||||||
|
k8up.io/backupcommand: >-
|
||||||
|
python3 -c "import sqlite3, sys;
|
||||||
|
con = sqlite3.connect('file:/data/terdut.db?mode=ro', uri=True);
|
||||||
|
con.execute('BEGIN');
|
||||||
|
users = con.execute('SELECT count(*) FROM users').fetchone()[0];
|
||||||
|
out = chr(10).join(con.iterdump()) + chr(10);
|
||||||
|
(users > 0 and out.rstrip().endswith('COMMIT;'))
|
||||||
|
or sys.exit('terdut: db dump failed sanity checks');
|
||||||
|
sys.stdout.write(out)"
|
||||||
|
k8up.io/file-extension: ".sql"
|
||||||
|
k8up.io/backup: "true"
|
||||||
|
{{- end }}
|
||||||
spec:
|
spec:
|
||||||
enableServiceLinks: false
|
enableServiceLinks: false
|
||||||
containers:
|
containers:
|
||||||
@@ -29,6 +69,33 @@ spec:
|
|||||||
value: ":{{ .Values.service.port }}"
|
value: ":{{ .Values.service.port }}"
|
||||||
- name: TERDUT_DB_PATH
|
- name: TERDUT_DB_PATH
|
||||||
value: "/data/terdut.db"
|
value: "/data/terdut.db"
|
||||||
|
- name: TERDUT_STALE_AFTER
|
||||||
|
value: "{{ .Values.sweeper.staleAfter }}"
|
||||||
|
- name: TERDUT_ARCHIVE_AFTER
|
||||||
|
value: "{{ .Values.sweeper.archiveAfter }}"
|
||||||
|
- name: TERDUT_DEADMAN_MATCHERS
|
||||||
|
value: "{{ .Values.deadman.matchers }}"
|
||||||
|
- name: TERDUT_DEADMAN_TIMEOUT
|
||||||
|
value: "{{ .Values.deadman.timeout }}"
|
||||||
|
- name: TERDUT_DEADMAN_SEVERITY
|
||||||
|
value: "{{ .Values.deadman.severity }}"
|
||||||
|
{{- if .Values.notify.ntfyUrl }}
|
||||||
|
- name: TERDUT_NTFY_URL
|
||||||
|
value: "{{ .Values.notify.ntfyUrl }}"
|
||||||
|
- name: TERDUT_NTFY_FALLBACK_TOPIC
|
||||||
|
value: "{{ .Values.notify.fallbackTopic }}"
|
||||||
|
- name: TERDUT_NOTIFY_REPEAT
|
||||||
|
value: "{{ .Values.notify.repeatEvery }}"
|
||||||
|
- name: TERDUT_PUBLIC_URL
|
||||||
|
value: "{{ .Values.notify.publicUrl | default (printf "https://%s" .Values.networking.hostname) }}"
|
||||||
|
{{- if .Values.notify.tokenSecret.name }}
|
||||||
|
- name: TERDUT_NTFY_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: {{ .Values.notify.tokenSecret.name }}
|
||||||
|
key: {{ .Values.notify.tokenSecret.key }}
|
||||||
|
{{- end }}
|
||||||
|
{{- end }}
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: data
|
- name: data
|
||||||
mountPath: /data
|
mountPath: /data
|
||||||
@@ -42,6 +109,25 @@ spec:
|
|||||||
path: /healthz
|
path: /healthz
|
||||||
port: http
|
port: http
|
||||||
initialDelaySeconds: 5
|
initialDelaySeconds: 5
|
||||||
|
|
||||||
|
{{- if .Values.backupSidecar.enabled }}
|
||||||
|
# Idle sidecar. It exists only so k8up has a container with a sqlite3
|
||||||
|
# module to exec the backupcommand in. Mounted read-write on purpose:
|
||||||
|
# see the note on the backupcommand annotation above.
|
||||||
|
- name: backup
|
||||||
|
image: "{{ .Values.backupSidecar.image.repository }}:{{ .Values.backupSidecar.image.tag }}"
|
||||||
|
imagePullPolicy: {{ .Values.backupSidecar.image.pullPolicy }}
|
||||||
|
command: ["sleep", "infinity"]
|
||||||
|
volumeMounts:
|
||||||
|
- name: data
|
||||||
|
mountPath: /data
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: "16Mi"
|
||||||
|
cpu: "10m"
|
||||||
|
limits:
|
||||||
|
memory: "64Mi"
|
||||||
|
{{- end }}
|
||||||
volumes:
|
volumes:
|
||||||
- name: data
|
- name: data
|
||||||
persistentVolumeClaim:
|
persistentVolumeClaim:
|
||||||
|
|||||||
@@ -9,6 +9,9 @@ spec:
|
|||||||
parentRefs:
|
parentRefs:
|
||||||
- name: envoy-main
|
- name: envoy-main
|
||||||
namespace: envoy-gateway-system
|
namespace: envoy-gateway-system
|
||||||
|
{{- with .Values.networking.listener }}
|
||||||
|
sectionName: {{ . | quote }}
|
||||||
|
{{- end }}
|
||||||
rules:
|
rules:
|
||||||
- backendRefs:
|
- backendRefs:
|
||||||
- name: {{ include "terdut-server.fullname" . }}
|
- name: {{ include "terdut-server.fullname" . }}
|
||||||
|
|||||||
@@ -1,9 +1,13 @@
|
|||||||
networking:
|
networking:
|
||||||
hostname: "terdut.example.com"
|
hostname: "terdut.example.com"
|
||||||
servicePort: 8080
|
servicePort: 8080
|
||||||
|
# Gateway listener to bind the HTTPRoute to. Empty attaches to every matching
|
||||||
|
# listener, including plaintext HTTP. Set this to the name of the HTTPS
|
||||||
|
# listener to serve the API over TLS only.
|
||||||
|
listener: ""
|
||||||
|
|
||||||
image:
|
image:
|
||||||
repository: ghcr.io/yeniklas/terdut-server
|
repository: git.ryuvia.com/niklas/terdut-server
|
||||||
tag: "latest"
|
tag: "latest"
|
||||||
pullPolicy: IfNotPresent
|
pullPolicy: IfNotPresent
|
||||||
|
|
||||||
@@ -15,6 +19,87 @@ service:
|
|||||||
type: ClusterIP
|
type: ClusterIP
|
||||||
port: 8080
|
port: 8080
|
||||||
|
|
||||||
|
sweeper:
|
||||||
|
# How long a firing alert may go without a refreshing webhook before it is
|
||||||
|
# treated as resolved. Must exceed your Alertmanager repeat_interval.
|
||||||
|
staleAfter: 6h
|
||||||
|
# How long a resolved alert stays in the default list before auto-archiving.
|
||||||
|
archiveAfter: 168h
|
||||||
|
|
||||||
|
# Alerts treated as dead man's switches: receiving one opens no incident, and
|
||||||
|
# the absence of one does. The Watchdog alert kube-prometheus-stack ships is
|
||||||
|
# exactly this — an always-firing alert whose only value is something noticing
|
||||||
|
# when it stops.
|
||||||
|
deadman:
|
||||||
|
# Which alerts to treat as heartbeats. ";" separates matchers, "," separates
|
||||||
|
# the label conditions within one, "=" is exact equality. Every matcher must
|
||||||
|
# name an alertname:
|
||||||
|
# alertname=Watchdog,cluster=prod; alertname=EdgeHeartbeat
|
||||||
|
# Each distinct label set is watched independently, so two clusters sending
|
||||||
|
# the same alertname are two switches and a live one cannot mask a dead one.
|
||||||
|
matchers: "alertname=Watchdog"
|
||||||
|
# How long a heartbeat may go unheard before its switch is declared dead.
|
||||||
|
#
|
||||||
|
# This must be SHORTER than the Alertmanager repeat_interval of the route
|
||||||
|
# carrying the heartbeat — the opposite of sweeper.staleAfter. The default
|
||||||
|
# repeat_interval of 4h (12h in many setups) makes for a useless dead man's
|
||||||
|
# switch, so give the heartbeat a route of its own:
|
||||||
|
#
|
||||||
|
# - matchers: [ 'alertname = "Watchdog"' ]
|
||||||
|
# receiver: terdut
|
||||||
|
# group_wait: 0s
|
||||||
|
# group_interval: 1m
|
||||||
|
# repeat_interval: 1m
|
||||||
|
#
|
||||||
|
# That delivers every 2m rather than every 1m: a group is only reconsidered
|
||||||
|
# each group_interval, and at exactly one elapsed interval repeat_interval has
|
||||||
|
# not quite passed, so equal values give 2x. Fine against 15m; use
|
||||||
|
# group_interval: 30s if you want a true 1m.
|
||||||
|
#
|
||||||
|
# Set to 0 to disable dead man's switch handling entirely.
|
||||||
|
timeout: 15m
|
||||||
|
# Severity a dead man's switch incident opens at. These incidents have no
|
||||||
|
# member alerts to derive one from, and the heartbeat's own severity label is
|
||||||
|
# meaningless — Watchdog ships as "none". Only "critical" maps to the ntfy
|
||||||
|
# priority that overrides a phone's quiet hours.
|
||||||
|
severity: critical
|
||||||
|
|
||||||
|
notify:
|
||||||
|
# ntfy server that push notifications are published to, e.g.
|
||||||
|
# http://ntfy.ntfy.svc.cluster.local. Empty disables notifications entirely.
|
||||||
|
ntfyUrl: ""
|
||||||
|
# Topic used when nobody is on call today. Notifications sent here carry no
|
||||||
|
# Acknowledge button: the topic is shared, so there is no user to attribute an
|
||||||
|
# acknowledgement to. Leave empty to send nothing when the schedule is unset.
|
||||||
|
fallbackTopic: ""
|
||||||
|
# How long an incident may sit unacknowledged before it is paged again.
|
||||||
|
# Set to 0 to notify once and never repeat.
|
||||||
|
repeatEvery: 15m
|
||||||
|
# Base URL a phone uses to reach this server, for the link and the Acknowledge
|
||||||
|
# button inside a notification. Defaults to https://<networking.hostname>.
|
||||||
|
#
|
||||||
|
# The Acknowledge button is a POST to /api/notify/ack/{token} from the
|
||||||
|
# responder's phone, so that path has to stay publicly reachable — it is
|
||||||
|
# authorised by the scoped token in the URL, not by network placement.
|
||||||
|
publicUrl: ""
|
||||||
|
# Optional bearer token for an access-controlled ntfy, read from an existing
|
||||||
|
# Secret. Leave name empty for an open ntfy.
|
||||||
|
tokenSecret:
|
||||||
|
name: ""
|
||||||
|
key: token
|
||||||
|
|
||||||
|
# The server image is FROM scratch — just the binary, with no shell, no sqlite3
|
||||||
|
# and no python — so a k8up backupcommand cannot run in the app container. This
|
||||||
|
# idle sidecar shares the data volume and is selected with
|
||||||
|
# k8up.io/backupcommand-container. Only the stdlib sqlite3 module is used, so any
|
||||||
|
# python image works.
|
||||||
|
backupSidecar:
|
||||||
|
enabled: true
|
||||||
|
image:
|
||||||
|
repository: python
|
||||||
|
tag: "3.13-alpine"
|
||||||
|
pullPolicy: IfNotPresent
|
||||||
|
|
||||||
bootstrap:
|
bootstrap:
|
||||||
enabled: true
|
enabled: true
|
||||||
username: admin
|
username: admin
|
||||||
|
|||||||
+16
-5
@@ -8,9 +8,9 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/yeniklas/terdut-server/internal/api"
|
"git.ryuvia.com/niklas/terdut-server/internal/api"
|
||||||
"github.com/yeniklas/terdut-server/internal/config"
|
"git.ryuvia.com/niklas/terdut-server/internal/config"
|
||||||
"github.com/yeniklas/terdut-server/internal/db"
|
"git.ryuvia.com/niklas/terdut-server/internal/db"
|
||||||
)
|
)
|
||||||
|
|
||||||
var version = "dev"
|
var version = "dev"
|
||||||
@@ -28,7 +28,17 @@ func main() {
|
|||||||
log.Fatalf("migrate: %v", err)
|
log.Fatalf("migrate: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
router := api.NewRouter(database)
|
notify := api.NotifyConfig{
|
||||||
|
BaseURL: cfg.NtfyURL,
|
||||||
|
Token: cfg.NtfyToken,
|
||||||
|
FallbackTopic: cfg.NtfyFallbackTopic,
|
||||||
|
PublicURL: cfg.PublicURL,
|
||||||
|
RepeatEvery: cfg.NotifyRepeat,
|
||||||
|
}
|
||||||
|
|
||||||
|
deadman := api.ParseDeadmanConfig(cfg.DeadmanMatchers, cfg.DeadmanTimeout, cfg.DeadmanSeverity)
|
||||||
|
|
||||||
|
router := api.NewRouter(database, notify, deadman)
|
||||||
|
|
||||||
srv := &http.Server{
|
srv := &http.Server{
|
||||||
Addr: cfg.Addr,
|
Addr: cfg.Addr,
|
||||||
@@ -41,7 +51,8 @@ func main() {
|
|||||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
go api.StartArchiver(ctx, database, cfg.ArchiveAfter)
|
go api.StartArchiver(ctx, database, cfg.ArchiveAfter, cfg.StaleAfter, deadman, notify)
|
||||||
|
go api.StartNotifier(ctx, database, notify)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
log.Printf("terdut-server %s listening on %s", version, cfg.Addr)
|
log.Printf("terdut-server %s listening on %s", version, cfg.Addr)
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
module github.com/yeniklas/terdut-server
|
module git.ryuvia.com/niklas/terdut-server
|
||||||
|
|
||||||
go 1.25.9
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/go-chi/chi/v5 v5.2.5
|
github.com/go-chi/chi/v5 v5.2.5
|
||||||
|
golang.org/x/crypto v0.57.0
|
||||||
modernc.org/sqlite v1.50.1
|
modernc.org/sqlite v1.50.1
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -13,7 +14,7 @@ require (
|
|||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
golang.org/x/sys v0.42.0 // indirect
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
modernc.org/libc v1.72.3 // indirect
|
modernc.org/libc v1.72.3 // indirect
|
||||||
modernc.org/mathutil v1.7.1 // indirect
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
modernc.org/memory v1.11.0 // indirect
|
modernc.org/memory v1.11.0 // indirect
|
||||||
|
|||||||
@@ -14,13 +14,15 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF
|
|||||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||||
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
|
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
|
||||||
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
|
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
|
||||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||||
modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY=
|
modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY=
|
||||||
|
|||||||
+361
-36
@@ -1,6 +1,7 @@
|
|||||||
package api
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"log"
|
"log"
|
||||||
@@ -8,11 +9,32 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Values for alerts.resolution_source, recording why an alert left the firing
|
||||||
|
// state: a real Alertmanager notification, or inference by the sweeper.
|
||||||
|
const (
|
||||||
|
resolutionAlertmanager = "alertmanager"
|
||||||
|
resolutionExpiry = "expiry"
|
||||||
|
|
||||||
|
// resolutionDeadman marks a heartbeat the dead man's switch sweeper declared
|
||||||
|
// dead. Distinct from expiry because it is load-bearing, not just
|
||||||
|
// descriptive: it is the one resolution the ingest upsert will let a
|
||||||
|
// same-instance re-fire undo, so a switch that comes back can be heard.
|
||||||
|
resolutionDeadman = "deadman"
|
||||||
|
)
|
||||||
|
|
||||||
// amPayload mirrors the Alertmanager webhook v4 payload.
|
// amPayload mirrors the Alertmanager webhook v4 payload.
|
||||||
type amPayload struct {
|
type amPayload struct {
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
Alerts []amAlert `json:"alerts"`
|
|
||||||
|
// GroupKey and GroupLabels are how alerts get correlated into incidents.
|
||||||
|
// Alertmanager has already done the grouping work according to the group_by
|
||||||
|
// routing tree the operator configured, so we adopt its answer instead of
|
||||||
|
// inventing a second grouping scheme here.
|
||||||
|
GroupKey string `json:"groupKey"`
|
||||||
|
GroupLabels map[string]string `json:"groupLabels"`
|
||||||
|
|
||||||
|
Alerts []amAlert `json:"alerts"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type amAlert struct {
|
type amAlert struct {
|
||||||
@@ -25,7 +47,30 @@ type amAlert struct {
|
|||||||
Fingerprint string `json:"fingerprint"`
|
Fingerprint string `json:"fingerprint"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func handleAlertmanagerWebhook(db *sql.DB) http.HandlerFunc {
|
// ingested records what actually happened to one alert of a payload, which is
|
||||||
|
// what decides whether an incident opens.
|
||||||
|
type ingested struct {
|
||||||
|
id int64
|
||||||
|
name string
|
||||||
|
firing bool
|
||||||
|
|
||||||
|
// newOccurrence marks an alert that transitioned *into* firing: a
|
||||||
|
// fingerprint we had never seen, a newer startsAt, or a resolved alert that
|
||||||
|
// started again. A repeat_interval re-send of an already-firing alert is
|
||||||
|
// none of these, which is what keeps a manually resolved incident closed.
|
||||||
|
newOccurrence bool
|
||||||
|
|
||||||
|
// justResolved marks the firing → resolved edge, worth a timeline entry.
|
||||||
|
justResolved bool
|
||||||
|
|
||||||
|
// deadman marks a heartbeat: an alert whose arrival means everything is
|
||||||
|
// fine. It is stored like any other alert — received_at is the heartbeat —
|
||||||
|
// but it never reaches an incident. Its absence is what opens one, which
|
||||||
|
// sweepDeadman decides later and elsewhere.
|
||||||
|
deadman bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleAlertmanagerWebhook(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
var payload amPayload
|
var payload amPayload
|
||||||
if err := decodeJSON(r, &payload); err != nil {
|
if err := decodeJSON(r, &payload); err != nil {
|
||||||
@@ -33,40 +78,320 @@ func handleAlertmanagerWebhook(db *sql.DB) http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now().Unix()
|
// Alertmanager retries anything that is not 2xx, and a retry of a payload
|
||||||
for _, a := range payload.Alerts {
|
// we failed to store is more useful than an error it cannot act on — so
|
||||||
name := a.Labels["alertname"]
|
// failures are logged, not surfaced.
|
||||||
labelsJSON, _ := json.Marshal(a.Labels)
|
if err := ingest(r.Context(), db, notify, deadman, payload); err != nil {
|
||||||
annotationsJSON, _ := json.Marshal(a.Annotations)
|
log.Printf("webhook ingest (group %q): %v", payload.GroupKey, err)
|
||||||
|
|
||||||
// Alertmanager uses zero time ("0001-01-01T00:00:00Z") to mean "still firing".
|
|
||||||
var endsAtUnix *int64
|
|
||||||
if a.EndsAt.Year() > 1 {
|
|
||||||
t := a.EndsAt.Unix()
|
|
||||||
endsAtUnix = &t
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := db.ExecContext(r.Context(), `
|
|
||||||
INSERT INTO alerts
|
|
||||||
(fingerprint, name, status, labels, annotations, starts_at, ends_at, generator_url, received_at)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
||||||
ON CONFLICT(fingerprint) DO UPDATE SET
|
|
||||||
status = excluded.status,
|
|
||||||
labels = excluded.labels,
|
|
||||||
annotations = excluded.annotations,
|
|
||||||
ends_at = excluded.ends_at,
|
|
||||||
generator_url = excluded.generator_url,
|
|
||||||
received_at = excluded.received_at`,
|
|
||||||
a.Fingerprint, name, a.Status,
|
|
||||||
string(labelsJSON), string(annotationsJSON),
|
|
||||||
a.StartsAt.Unix(), endsAtUnix,
|
|
||||||
a.GeneratorURL, now,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("upsert alert %s: %v", a.Fingerprint, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ingest stores a payload's alerts and reconciles the incident for its group.
|
||||||
|
// The whole payload is one transaction: an incident that opened but whose alerts
|
||||||
|
// failed to link would be a work item nobody could act on.
|
||||||
|
func ingest(ctx context.Context, db *sql.DB, notify NotifyConfig, deadman DeadmanConfig, payload amPayload) error {
|
||||||
|
tx, err := db.BeginTx(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer tx.Rollback() //nolint:errcheck
|
||||||
|
|
||||||
|
accepted, err := upsertAlerts(ctx, tx, deadman, payload.Alerts)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// touched collects every incident this payload affected, so severity and the
|
||||||
|
// resolution cascade are recomputed once per incident at the end.
|
||||||
|
touched := map[int64]bool{}
|
||||||
|
|
||||||
|
incidentID, err := incidentForGroup(ctx, tx, notify, payload, accepted)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if incidentID != 0 {
|
||||||
|
touched[incidentID] = true
|
||||||
|
for _, a := range accepted {
|
||||||
|
if !a.firing || a.deadman {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := linkAlert(ctx, tx, incidentID, a.id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, a := range accepted {
|
||||||
|
if !a.justResolved || a.deadman {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
id, err := openIncidentForAlert(ctx, tx, a.id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if id == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
touched[id] = true
|
||||||
|
alertID := a.id
|
||||||
|
if err := logEvent(ctx, tx, id, evAlertResolved, nil, &alertID, nil); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for id := range touched {
|
||||||
|
if err := refreshSeverity(ctx, tx, id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := resolveIfSettled(ctx, tx, id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
// upsertAlerts stores each alert of a payload and reports what changed. Payloads
|
||||||
|
// the ordering guard rejected are left out entirely.
|
||||||
|
func upsertAlerts(ctx context.Context, tx *sql.Tx, deadman DeadmanConfig, alerts []amAlert) ([]ingested, error) {
|
||||||
|
now := time.Now().Unix()
|
||||||
|
accepted := make([]ingested, 0, len(alerts))
|
||||||
|
|
||||||
|
for _, a := range alerts {
|
||||||
|
name := a.Labels["alertname"]
|
||||||
|
labelsJSON, _ := json.Marshal(a.Labels)
|
||||||
|
annotationsJSON, _ := json.Marshal(a.Annotations)
|
||||||
|
|
||||||
|
// The stored state has to be read before the upsert overwrites it: it is
|
||||||
|
// the only way to tell a genuine new occurrence from a re-send.
|
||||||
|
var prevStatus string
|
||||||
|
var prevStartsAt int64
|
||||||
|
existed := true
|
||||||
|
switch err := tx.QueryRowContext(ctx,
|
||||||
|
"SELECT status, starts_at FROM alerts WHERE fingerprint = ?", a.Fingerprint,
|
||||||
|
).Scan(&prevStatus, &prevStartsAt); {
|
||||||
|
case err == sql.ErrNoRows:
|
||||||
|
existed = false
|
||||||
|
case err != nil:
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Zero time ("0001-01-01T00:00:00Z") means "no end known" — that is the
|
||||||
|
// convention of Alertmanager's ingest API. Outgoing notifications
|
||||||
|
// normally carry a real future endsAt instead, which is the watermark
|
||||||
|
// the sweeper uses to expire alerts that stop being refreshed.
|
||||||
|
var endsAtUnix *int64
|
||||||
|
if a.EndsAt.Year() > 1 {
|
||||||
|
t := a.EndsAt.Unix()
|
||||||
|
endsAtUnix = &t
|
||||||
|
}
|
||||||
|
|
||||||
|
var resolutionSource *string
|
||||||
|
if a.Status == "resolved" {
|
||||||
|
s := resolutionAlertmanager
|
||||||
|
resolutionSource = &s
|
||||||
|
}
|
||||||
|
|
||||||
|
// The WHERE clause discards payloads that describe an alert instance
|
||||||
|
// older than the stored one. Alertmanager retries failed notifications,
|
||||||
|
// so a stale firing retry can arrive after the resolved one; it carries
|
||||||
|
// the same startsAt, whereas a genuine re-fire carries a newer one.
|
||||||
|
// Within a single instance, resolution is terminal — with one exception.
|
||||||
|
//
|
||||||
|
// A resolution this server synthesised for a dead man's switch is not
|
||||||
|
// Alertmanager's word that the instance ended; it is our inference from
|
||||||
|
// silence. The heartbeat that proves us wrong carries the unchanged
|
||||||
|
// startsAt of an alert that never stopped firing, so without the
|
||||||
|
// exemption a switch could go dead exactly once and never be heard from
|
||||||
|
// again. Scoped to 'deadman' so no resolution anybody else wrote can be
|
||||||
|
// undone by a stale retry.
|
||||||
|
if _, err := tx.ExecContext(ctx, `
|
||||||
|
INSERT INTO alerts
|
||||||
|
(fingerprint, name, status, labels, annotations, starts_at, ends_at,
|
||||||
|
generator_url, received_at, resolution_source)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(fingerprint) DO UPDATE SET
|
||||||
|
status = excluded.status,
|
||||||
|
labels = excluded.labels,
|
||||||
|
annotations = excluded.annotations,
|
||||||
|
starts_at = excluded.starts_at,
|
||||||
|
ends_at = excluded.ends_at,
|
||||||
|
generator_url = excluded.generator_url,
|
||||||
|
-- Load-bearing: advancing received_at on every accepted
|
||||||
|
-- payload, re-sends included, is the documented liveness
|
||||||
|
-- heartbeat clients and the sweeper both read. Removing it
|
||||||
|
-- is a breaking API change — see models.Alert.ReceivedAt.
|
||||||
|
received_at = excluded.received_at,
|
||||||
|
resolution_source = excluded.resolution_source,
|
||||||
|
-- A re-fire makes the alert current again, so it leaves the archive.
|
||||||
|
archived_at = CASE WHEN excluded.status = 'firing'
|
||||||
|
THEN NULL ELSE alerts.archived_at END
|
||||||
|
WHERE excluded.starts_at > alerts.starts_at
|
||||||
|
OR (excluded.starts_at = alerts.starts_at
|
||||||
|
AND (alerts.resolution_source = '`+resolutionDeadman+`'
|
||||||
|
OR NOT (alerts.status = 'resolved' AND excluded.status = 'firing')))`,
|
||||||
|
a.Fingerprint, name, a.Status,
|
||||||
|
string(labelsJSON), string(annotationsJSON),
|
||||||
|
a.StartsAt.Unix(), endsAtUnix,
|
||||||
|
a.GeneratorURL, now, resolutionSource,
|
||||||
|
); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var id int64
|
||||||
|
var curStatus string
|
||||||
|
var curStartsAt int64
|
||||||
|
if err := tx.QueryRowContext(ctx,
|
||||||
|
"SELECT id, status, starts_at FROM alerts WHERE fingerprint = ?", a.Fingerprint,
|
||||||
|
).Scan(&id, &curStatus, &curStartsAt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The upsert copies status and starts_at straight from the payload, so a
|
||||||
|
// row that does not match it is one the ordering guard rejected. A
|
||||||
|
// discarded payload describes a past instance and must not touch the
|
||||||
|
// incident state either.
|
||||||
|
if existed && (curStatus != a.Status || curStartsAt != a.StartsAt.Unix()) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
firing := a.Status == "firing"
|
||||||
|
accepted = append(accepted, ingested{
|
||||||
|
id: id,
|
||||||
|
name: name,
|
||||||
|
firing: firing,
|
||||||
|
newOccurrence: firing && (!existed || a.StartsAt.Unix() > prevStartsAt || prevStatus == "resolved"),
|
||||||
|
justResolved: !firing && existed && prevStatus == "firing",
|
||||||
|
deadman: deadman.isDeadman(a.Labels),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return accepted, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// incidentForGroup returns the open incident that this payload's firing alerts
|
||||||
|
// belong to, opening one if the group has none. It returns 0 when the payload
|
||||||
|
// warrants no incident at all.
|
||||||
|
//
|
||||||
|
// The rule that matters: a group with no open incident gets a new one only if
|
||||||
|
// something actually started firing. Without that, a manually resolved incident
|
||||||
|
// would reappear on the next repeat_interval re-send of an alert that never
|
||||||
|
// stopped, and manual resolution would be meaningless.
|
||||||
|
//
|
||||||
|
// Heartbeats do not count as anything here. A group of nothing but dead man's
|
||||||
|
// switch alerts opens no incident at all, and a mixed group gets an incident for
|
||||||
|
// its real alerts only.
|
||||||
|
func incidentForGroup(ctx context.Context, tx *sql.Tx, notify NotifyConfig, payload amPayload, accepted []ingested) (int64, error) {
|
||||||
|
var firstName string
|
||||||
|
anyFiring, anyNew := false, false
|
||||||
|
for _, a := range accepted {
|
||||||
|
if a.deadman {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if a.firing {
|
||||||
|
if !anyFiring {
|
||||||
|
firstName = a.name
|
||||||
|
}
|
||||||
|
anyFiring = true
|
||||||
|
}
|
||||||
|
if a.newOccurrence {
|
||||||
|
anyNew = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !anyFiring {
|
||||||
|
// A payload of nothing but resolutions never opens an incident.
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
groupKey := payload.GroupKey
|
||||||
|
if groupKey == "" {
|
||||||
|
// Alertmanager always sends groupKey; a sender that does not still gets
|
||||||
|
// one incident per alert name rather than one giant shared incident.
|
||||||
|
groupKey = "groupless:" + firstName
|
||||||
|
}
|
||||||
|
|
||||||
|
var id int64
|
||||||
|
switch err := tx.QueryRowContext(ctx,
|
||||||
|
"SELECT id FROM incidents WHERE group_key = ? AND resolved_at IS NULL", groupKey,
|
||||||
|
).Scan(&id); {
|
||||||
|
case err == nil:
|
||||||
|
return id, nil
|
||||||
|
case err != sql.ErrNoRows:
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !anyNew {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
return openIncident(ctx, tx, notify, groupKey,
|
||||||
|
incidentTitle(payload.GroupLabels, firstName), payload.GroupLabels, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// openIncident creates an incident and assigns it to whoever is on call today,
|
||||||
|
// which is the point at which the schedule stops being decorative.
|
||||||
|
//
|
||||||
|
// The one place an incident is born, for both of the things that can raise one:
|
||||||
|
// the webhook, inside its transaction, and the dead man's switch sweeper, inside
|
||||||
|
// its own. Hence the querier rather than a *sql.Tx. A nil severity leaves the
|
||||||
|
// column for refreshSeverity to fill from the member alerts; the sweeper passes
|
||||||
|
// one because its incidents have no members to derive it from.
|
||||||
|
func openIncident(ctx context.Context, q querier, notify NotifyConfig, groupKey, title string, groupLabels map[string]string, severity *string) (int64, error) {
|
||||||
|
onCall, err := currentOnCall(ctx, q)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
labelsJSON, _ := json.Marshal(groupLabels)
|
||||||
|
if groupLabels == nil {
|
||||||
|
labelsJSON = []byte("{}")
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := q.ExecContext(ctx, `
|
||||||
|
INSERT INTO incidents (group_key, title, group_labels, status, severity, triggered_at, assigned_to)
|
||||||
|
VALUES (?, ?, ?, 'triggered', ?, ?, ?)`,
|
||||||
|
groupKey, title, string(labelsJSON), severity,
|
||||||
|
time.Now().Unix(), onCall)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
id, err := res.LastInsertId()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := logEvent(ctx, q, id, evTriggered, nil, nil, nil); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if onCall != nil {
|
||||||
|
// On an "assigned" event user_id is the assignee, not the actor.
|
||||||
|
if err := logEvent(ctx, q, id, evAssigned, onCall, nil, nil); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Queue the page, but do not send it here: this runs inside a transaction on
|
||||||
|
// a single-connection pool, so an HTTP call would hold up every other
|
||||||
|
// request. The notifier picks the row up within a tick.
|
||||||
|
if err := enqueueOpened(ctx, q, notify, id, onCall); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// linkAlert adds an alert to an incident, emitting a timeline entry only the
|
||||||
|
// first time. Re-sends of an already-linked alert are silent.
|
||||||
|
func linkAlert(ctx context.Context, tx *sql.Tx, incidentID, alertID int64) error {
|
||||||
|
res, err := tx.ExecContext(ctx, `
|
||||||
|
INSERT OR IGNORE INTO incident_alerts (incident_id, alert_id, added_at)
|
||||||
|
VALUES (?, ?, ?)`, incidentID, alertID, time.Now().Unix())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return logEvent(ctx, tx, incidentID, evAlertAdded, nil, &alertID, nil)
|
||||||
|
}
|
||||||
|
|||||||
+25
-118
@@ -10,20 +10,26 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/models"
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/yeniklas/terdut-server/internal/models"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// alertSelectFrom is the shared SELECT … FROM … clause used by all alert queries.
|
// alertSelectFrom is the shared SELECT … FROM … clause used by all alert queries.
|
||||||
// It LEFT JOINs users so acknowledged_by username is always available.
|
// The subquery resolves the alert's most recent incident: membership is kept in
|
||||||
|
// incident_alerts rather than as a column here, because one alert row is reused
|
||||||
|
// across occurrences and belongs to a different incident each time.
|
||||||
const alertSelectFrom = `
|
const alertSelectFrom = `
|
||||||
SELECT a.id, a.fingerprint, a.name, a.status,
|
SELECT a.id, a.fingerprint, a.name, a.status,
|
||||||
a.labels, a.annotations,
|
a.labels, a.annotations,
|
||||||
a.starts_at, a.ends_at, a.generator_url, a.received_at,
|
a.starts_at, a.ends_at, a.generator_url, a.received_at,
|
||||||
a.acknowledged_by, a.acknowledged_at, u.username,
|
(SELECT ia.incident_id
|
||||||
a.archived_at
|
FROM incident_alerts ia
|
||||||
FROM alerts a
|
JOIN incidents i ON i.id = ia.incident_id
|
||||||
LEFT JOIN users u ON u.id = a.acknowledged_by`
|
WHERE ia.alert_id = a.id
|
||||||
|
ORDER BY i.triggered_at DESC, i.id DESC
|
||||||
|
LIMIT 1),
|
||||||
|
a.resolution_source, a.archived_at
|
||||||
|
FROM alerts a`
|
||||||
|
|
||||||
func handleListAlerts(db *sql.DB) http.HandlerFunc {
|
func handleListAlerts(db *sql.DB) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -45,6 +51,12 @@ func handleListAlerts(db *sql.DB) http.HandlerFunc {
|
|||||||
} else {
|
} else {
|
||||||
where = append(where, "a.archived_at IS NULL")
|
where = append(where, "a.archived_at IS NULL")
|
||||||
}
|
}
|
||||||
|
if incidentID := q.Get("incident_id"); incidentID != "" {
|
||||||
|
if n, err := strconv.ParseInt(incidentID, 10, 64); err == nil {
|
||||||
|
where = append(where, "a.id IN (SELECT alert_id FROM incident_alerts WHERE incident_id = ?)")
|
||||||
|
args = append(args, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if from := q.Get("from"); from != "" {
|
if from := q.Get("from"); from != "" {
|
||||||
if t, err := time.Parse("2006-01-02", from); err == nil {
|
if t, err := time.Parse("2006-01-02", from); err == nil {
|
||||||
@@ -114,55 +126,7 @@ func handleGetAlert(db *sql.DB) http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func handleAcknowledge(db *sql.DB) http.HandlerFunc {
|
// fetchAlert loads a single alert by ID using the shared query.
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusBadRequest, errResp("invalid alert id"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
user, _ := userFromContext(r.Context())
|
|
||||||
|
|
||||||
res, err := db.ExecContext(r.Context(),
|
|
||||||
"UPDATE alerts SET acknowledged_by = ?, acknowledged_at = ? WHERE id = ?",
|
|
||||||
user.ID, time.Now().Unix(), id)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if n, _ := res.RowsAffected(); n == 0 {
|
|
||||||
respond(w, http.StatusNotFound, errResp("alert not found"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
a, _ := fetchAlert(r.Context(), db, id)
|
|
||||||
respond(w, http.StatusOK, a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func handleUnacknowledge(db *sql.DB) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusBadRequest, errResp("invalid alert id"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := db.ExecContext(r.Context(),
|
|
||||||
"UPDATE alerts SET acknowledged_by = NULL, acknowledged_at = NULL WHERE id = ?", id)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if n, _ := res.RowsAffected(); n == 0 {
|
|
||||||
respond(w, http.StatusNotFound, errResp("alert not found"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchAlert loads a single alert by ID using the shared JOIN query.
|
|
||||||
func fetchAlert(ctx context.Context, db *sql.DB, id int64) (models.Alert, error) {
|
func fetchAlert(ctx context.Context, db *sql.DB, id int64) (models.Alert, error) {
|
||||||
return scanAlert(db.QueryRowContext(ctx, alertSelectFrom+" WHERE a.id = ?", id))
|
return scanAlert(db.QueryRowContext(ctx, alertSelectFrom+" WHERE a.id = ?", id))
|
||||||
}
|
}
|
||||||
@@ -176,81 +140,24 @@ func scanAlert(s scanner) (models.Alert, error) {
|
|||||||
var a models.Alert
|
var a models.Alert
|
||||||
var labelsJSON, annotationsJSON string
|
var labelsJSON, annotationsJSON string
|
||||||
var startsAtUnix, receivedAtUnix int64
|
var startsAtUnix, receivedAtUnix int64
|
||||||
var endsAtUnix, ackAtUnix, archivedAtUnix *int64
|
var endsAtUnix, archivedAtUnix *int64
|
||||||
var ackByID *int64
|
|
||||||
var ackByUser *string
|
|
||||||
|
|
||||||
if err := s.Scan(
|
if err := s.Scan(
|
||||||
&a.ID, &a.Fingerprint, &a.Name, &a.Status,
|
&a.ID, &a.Fingerprint, &a.Name, &a.Status,
|
||||||
&labelsJSON, &annotationsJSON,
|
&labelsJSON, &annotationsJSON,
|
||||||
&startsAtUnix, &endsAtUnix,
|
&startsAtUnix, &endsAtUnix,
|
||||||
&a.GeneratorURL, &receivedAtUnix,
|
&a.GeneratorURL, &receivedAtUnix,
|
||||||
&ackByID, &ackAtUnix, &ackByUser,
|
&a.IncidentID,
|
||||||
&archivedAtUnix,
|
&a.ResolutionSource, &archivedAtUnix,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
return a, err
|
return a, err
|
||||||
}
|
}
|
||||||
|
|
||||||
json.Unmarshal([]byte(labelsJSON), &a.Labels) //nolint:errcheck
|
json.Unmarshal([]byte(labelsJSON), &a.Labels) //nolint:errcheck
|
||||||
json.Unmarshal([]byte(annotationsJSON), &a.Annotations) //nolint:errcheck
|
json.Unmarshal([]byte(annotationsJSON), &a.Annotations) //nolint:errcheck
|
||||||
a.StartsAt = time.Unix(startsAtUnix, 0).UTC()
|
a.StartsAt = time.Unix(startsAtUnix, 0).UTC()
|
||||||
a.ReceivedAt = time.Unix(receivedAtUnix, 0).UTC()
|
a.ReceivedAt = time.Unix(receivedAtUnix, 0).UTC()
|
||||||
if endsAtUnix != nil {
|
a.EndsAt = unixPtr(endsAtUnix)
|
||||||
t := time.Unix(*endsAtUnix, 0).UTC()
|
a.ArchivedAt = unixPtr(archivedAtUnix)
|
||||||
a.EndsAt = &t
|
|
||||||
}
|
|
||||||
if ackByID != nil {
|
|
||||||
t := time.Unix(*ackAtUnix, 0).UTC()
|
|
||||||
a.AcknowledgedByID = ackByID
|
|
||||||
a.AcknowledgedByUser = ackByUser
|
|
||||||
a.AcknowledgedAt = &t
|
|
||||||
}
|
|
||||||
if archivedAtUnix != nil {
|
|
||||||
t := time.Unix(*archivedAtUnix, 0).UTC()
|
|
||||||
a.ArchivedAt = &t
|
|
||||||
}
|
|
||||||
return a, nil
|
return a, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func handleArchive(db *sql.DB) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusBadRequest, errResp("invalid alert id"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
res, err := db.ExecContext(r.Context(),
|
|
||||||
"UPDATE alerts SET archived_at = unixepoch() WHERE id = ?", id)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if n, _ := res.RowsAffected(); n == 0 {
|
|
||||||
respond(w, http.StatusNotFound, errResp("alert not found"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a, _ := fetchAlert(r.Context(), db, id)
|
|
||||||
respond(w, http.StatusOK, a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func handleUnarchive(db *sql.DB) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusBadRequest, errResp("invalid alert id"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
res, err := db.ExecContext(r.Context(),
|
|
||||||
"UPDATE alerts SET archived_at = NULL WHERE id = ?", id)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if n, _ := res.RowsAffected(); n == 0 {
|
|
||||||
respond(w, http.StatusNotFound, errResp("alert not found"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
+497
-109
@@ -2,25 +2,50 @@ package api_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/yeniklas/terdut-server/internal/api"
|
"git.ryuvia.com/niklas/terdut-server/internal/api"
|
||||||
"github.com/yeniklas/terdut-server/internal/db"
|
"git.ryuvia.com/niklas/terdut-server/internal/db"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ts wraps httptest.Server with a pre-bootstrapped API key.
|
// ts wraps httptest.Server with a pre-bootstrapped API key. db is exposed so
|
||||||
|
// tests can age rows directly — the sweeper's inputs are wall-clock timestamps.
|
||||||
type ts struct {
|
type ts struct {
|
||||||
*httptest.Server
|
*httptest.Server
|
||||||
key string
|
key string
|
||||||
|
db *sql.DB
|
||||||
|
notify api.NotifyConfig
|
||||||
|
deadman api.DeadmanConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
func newTS(t *testing.T) *ts {
|
// newTS builds a server over a fresh in-memory database. Notifications are off
|
||||||
|
// unless a NotifyConfig is passed, so tests that predate them are unaffected.
|
||||||
|
// Dead man's switches are off too — see newDeadmanTS.
|
||||||
|
func newTS(t *testing.T, notify ...api.NotifyConfig) *ts {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
var cfg api.NotifyConfig
|
||||||
|
if len(notify) > 0 {
|
||||||
|
cfg = notify[0]
|
||||||
|
}
|
||||||
|
return newDeadmanTS(t, api.DeadmanConfig{}, cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newDeadmanTS is newTS with dead man's switch handling configured.
|
||||||
|
func newDeadmanTS(t *testing.T, deadman api.DeadmanConfig, notify ...api.NotifyConfig) *ts {
|
||||||
|
t.Helper()
|
||||||
|
var cfg api.NotifyConfig
|
||||||
|
if len(notify) > 0 {
|
||||||
|
cfg = notify[0]
|
||||||
|
}
|
||||||
|
|
||||||
database, err := db.Open(":memory:")
|
database, err := db.Open(":memory:")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("open db: %v", err)
|
t.Fatalf("open db: %v", err)
|
||||||
@@ -28,7 +53,7 @@ func newTS(t *testing.T) *ts {
|
|||||||
if err := db.Migrate(database); err != nil {
|
if err := db.Migrate(database); err != nil {
|
||||||
t.Fatalf("migrate: %v", err)
|
t.Fatalf("migrate: %v", err)
|
||||||
}
|
}
|
||||||
srv := httptest.NewServer(api.NewRouter(database))
|
srv := httptest.NewServer(api.NewRouter(database, cfg, deadman))
|
||||||
t.Cleanup(func() { srv.Close(); database.Close() })
|
t.Cleanup(func() { srv.Close(); database.Close() })
|
||||||
|
|
||||||
body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"})
|
body, _ := json.Marshal(map[string]string{"username": "admin", "email": "admin@test.com"})
|
||||||
@@ -44,7 +69,50 @@ func newTS(t *testing.T) *ts {
|
|||||||
json.NewDecoder(resp.Body).Decode(&result)
|
json.NewDecoder(resp.Body).Decode(&result)
|
||||||
key := result["api_key"].(map[string]any)["key"].(string)
|
key := result["api_key"].(map[string]any)["key"].(string)
|
||||||
|
|
||||||
return &ts{Server: srv, key: key}
|
return &ts{Server: srv, key: key, db: database, notify: cfg, deadman: deadman}
|
||||||
|
}
|
||||||
|
|
||||||
|
// exec runs a statement against the test database.
|
||||||
|
func (s *ts) exec(t *testing.T, query string, args ...any) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := s.db.Exec(query, args...); err != nil {
|
||||||
|
t.Fatalf("exec %q: %v", query, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// alertRow reads the sweeper-relevant columns of one alert straight from the DB.
|
||||||
|
func (s *ts) alertRow(t *testing.T, fingerprint string) (status string, source *string, archivedAt *int64) {
|
||||||
|
t.Helper()
|
||||||
|
err := s.db.QueryRow(
|
||||||
|
"SELECT status, resolution_source, archived_at FROM alerts WHERE fingerprint = ?",
|
||||||
|
fingerprint).Scan(&status, &source, &archivedAt)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read alert %s: %v", fingerprint, err)
|
||||||
|
}
|
||||||
|
return status, source, archivedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
// alertTimes reads the timestamp columns that make up the received_at contract.
|
||||||
|
func (s *ts) alertTimes(t *testing.T, fingerprint string) (startsAt, receivedAt int64) {
|
||||||
|
t.Helper()
|
||||||
|
err := s.db.QueryRow(
|
||||||
|
"SELECT starts_at, received_at FROM alerts WHERE fingerprint = ?",
|
||||||
|
fingerprint).Scan(&startsAt, &receivedAt)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read alert times %s: %v", fingerprint, err)
|
||||||
|
}
|
||||||
|
return startsAt, receivedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
// alertEndsAt reads the nullable ends_at column of one alert.
|
||||||
|
func (s *ts) alertEndsAt(t *testing.T, fingerprint string) *int64 {
|
||||||
|
t.Helper()
|
||||||
|
var endsAt *int64
|
||||||
|
if err := s.db.QueryRow(
|
||||||
|
"SELECT ends_at FROM alerts WHERE fingerprint = ?", fingerprint).Scan(&endsAt); err != nil {
|
||||||
|
t.Fatalf("read ends_at %s: %v", fingerprint, err)
|
||||||
|
}
|
||||||
|
return endsAt
|
||||||
}
|
}
|
||||||
|
|
||||||
// req sends an authenticated request, optionally with a JSON body.
|
// req sends an authenticated request, optionally with a JSON body.
|
||||||
@@ -123,9 +191,22 @@ func TestBootstrap_SecondCallForbidden(t *testing.T) {
|
|||||||
// Alert upsert by fingerprint
|
// Alert upsert by fingerprint
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
func postWebhook(t *testing.T, s *ts, alerts []map[string]any) {
|
// postWebhook sends an Alertmanager v4 payload. groupKey is optional: omitting
|
||||||
|
// it exercises the fallback for senders that do not group, which is what most of
|
||||||
|
// these tests want.
|
||||||
|
func postWebhook(t *testing.T, s *ts, alerts []map[string]any, groupKey ...string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
payload := map[string]any{"version": "4", "status": "firing", "alerts": alerts}
|
payload := map[string]any{"version": "4", "status": "firing", "alerts": alerts}
|
||||||
|
if len(groupKey) > 0 {
|
||||||
|
payload["groupKey"] = groupKey[0]
|
||||||
|
// Alertmanager groups by alertname by default, so the group labels echo
|
||||||
|
// the first alert's name.
|
||||||
|
if len(alerts) > 0 {
|
||||||
|
if labels, ok := alerts[0]["labels"].(map[string]string); ok {
|
||||||
|
payload["groupLabels"] = map[string]string{"alertname": labels["alertname"]}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
data, _ := json.Marshal(payload)
|
data, _ := json.Marshal(payload)
|
||||||
resp, err := http.Post(s.URL+"/api/alertmanager/webhook", "application/json", bytes.NewReader(data))
|
resp, err := http.Post(s.URL+"/api/alertmanager/webhook", "application/json", bytes.NewReader(data))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -195,84 +276,6 @@ func TestAlertUpsert_DifferentFingerprintsStored(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Alert acknowledge
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
func TestAcknowledge(t *testing.T) {
|
|
||||||
s := newTS(t)
|
|
||||||
postWebhook(t, s, []map[string]any{{
|
|
||||||
"status": "firing", "labels": map[string]string{"alertname": "X"},
|
|
||||||
"annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z",
|
|
||||||
"endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "fp-ack",
|
|
||||||
}})
|
|
||||||
|
|
||||||
resp := s.req(t, http.MethodPost, "/api/alerts/1/acknowledge", nil)
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("acknowledge returned %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
var alert map[string]any
|
|
||||||
decode(t, resp, &alert)
|
|
||||||
if alert["acknowledged_by"] == nil {
|
|
||||||
t.Error("expected acknowledged_by to be set")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clear it.
|
|
||||||
resp = s.req(t, http.MethodDelete, "/api/alerts/1/acknowledge", nil)
|
|
||||||
if resp.StatusCode != http.StatusNoContent {
|
|
||||||
t.Errorf("unacknowledge returned %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp = s.req(t, http.MethodGet, "/api/alerts/1", nil)
|
|
||||||
var alert2 map[string]any
|
|
||||||
decode(t, resp, &alert2)
|
|
||||||
if alert2["acknowledged_by"] != nil {
|
|
||||||
t.Error("expected acknowledged_by to be cleared")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Comments — own-only deletion
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
func TestComment_DeleteOwnOnly(t *testing.T) {
|
|
||||||
s := newTS(t)
|
|
||||||
postWebhook(t, s, []map[string]any{{
|
|
||||||
"status": "firing", "labels": map[string]string{"alertname": "Y"},
|
|
||||||
"annotations": map[string]string{}, "startsAt": "2026-05-20T10:00:00Z",
|
|
||||||
"endsAt": "0001-01-01T00:00:00Z", "generatorURL": "", "fingerprint": "fp-comment",
|
|
||||||
}})
|
|
||||||
|
|
||||||
// Create a second user and their own key.
|
|
||||||
s.req(t, http.MethodPost, "/api/users",
|
|
||||||
map[string]string{"username": "alice", "email": "alice@test.com"})
|
|
||||||
keyResp := s.req(t, http.MethodPost, "/api/users/2/api-keys",
|
|
||||||
map[string]string{"name": "alice-key"})
|
|
||||||
var keyData map[string]any
|
|
||||||
decode(t, keyResp, &keyData)
|
|
||||||
aliceKey := keyData["key"].(string)
|
|
||||||
|
|
||||||
// Admin posts a comment.
|
|
||||||
s.req(t, http.MethodPost, "/api/alerts/1/comments",
|
|
||||||
map[string]string{"content": "admin note"})
|
|
||||||
|
|
||||||
// Alice tries to delete admin's comment (should 404).
|
|
||||||
req, _ := http.NewRequest(http.MethodDelete, s.URL+"/api/alerts/1/comments/1", nil)
|
|
||||||
req.Header.Set("Authorization", "Bearer "+aliceKey)
|
|
||||||
resp, _ := http.DefaultClient.Do(req)
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusNotFound {
|
|
||||||
t.Errorf("expected 404 when deleting another user's comment, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Admin deletes own comment (should 204).
|
|
||||||
resp = s.req(t, http.MethodDelete, "/api/alerts/1/comments/1", nil)
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusNoContent {
|
|
||||||
t.Errorf("expected 204 when deleting own comment, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Schedule conflict
|
// Schedule conflict
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -319,6 +322,125 @@ func TestSchedule_MultiDateRollbackOnConflict(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Schedule reassignment
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// addUser creates a second person to hand a shift to. The bootstrap user is
|
||||||
|
// admin, id 1.
|
||||||
|
func addUser(t *testing.T, s *ts, username string) {
|
||||||
|
t.Helper()
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/users",
|
||||||
|
map[string]any{"username": username, "email": username + "@test.com"})
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusCreated {
|
||||||
|
t.Fatalf("create user returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// scheduleHolder reports who is on call for one date, or "" for nobody.
|
||||||
|
func scheduleHolder(t *testing.T, s *ts, date string) string {
|
||||||
|
t.Helper()
|
||||||
|
var entries []map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/schedule?from="+date+"&to="+date, nil), &entries)
|
||||||
|
if len(entries) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return entries[0]["username"].(string)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Taking a day somebody else holds is possible, but only by asking for it.
|
||||||
|
func TestSchedule_ReplaceTakesAnAssignedDate(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
addUser(t, s, "alex")
|
||||||
|
|
||||||
|
s.req(t, http.MethodPost, "/api/schedule",
|
||||||
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}).Body.Close()
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/schedule",
|
||||||
|
map[string]any{"user_id": 2, "dates": []string{"2026-06-01"}, "replace": true})
|
||||||
|
if resp.StatusCode != http.StatusCreated {
|
||||||
|
t.Fatalf("expected replace to succeed, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
|
||||||
|
if got := scheduleHolder(t, s, "2026-06-01"); got != "alex" {
|
||||||
|
t.Errorf("expected alex to hold the day, got %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// One row, not two: two entries for a date would mean two people believing
|
||||||
|
// they are on call for it.
|
||||||
|
var entries []map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/schedule?from=2026-06-01&to=2026-06-01", nil), &entries)
|
||||||
|
if len(entries) != 1 {
|
||||||
|
t.Errorf("expected exactly one entry for the date, got %d", len(entries))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A week where only some days are taken is the case that was impossible before:
|
||||||
|
// the free days and the taken ones have to land together.
|
||||||
|
func TestSchedule_ReplaceMixedWeek(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
addUser(t, s, "alex")
|
||||||
|
|
||||||
|
s.req(t, http.MethodPost, "/api/schedule",
|
||||||
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-02", "2026-06-04"}}).Body.Close()
|
||||||
|
|
||||||
|
week := []string{"2026-06-01", "2026-06-02", "2026-06-03", "2026-06-04", "2026-06-05"}
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/schedule",
|
||||||
|
map[string]any{"user_id": 2, "dates": week, "replace": true})
|
||||||
|
if resp.StatusCode != http.StatusCreated {
|
||||||
|
t.Fatalf("expected the mixed week to succeed, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
|
||||||
|
for _, d := range week {
|
||||||
|
if got := scheduleHolder(t, s, d); got != "alex" {
|
||||||
|
t.Errorf("%s: expected alex, got %q", d, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Without replace the guard stands: nobody loses a shift by accident.
|
||||||
|
func TestSchedule_ReplaceDefaultsOff(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
addUser(t, s, "alex")
|
||||||
|
|
||||||
|
s.req(t, http.MethodPost, "/api/schedule",
|
||||||
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01"}}).Body.Close()
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/schedule",
|
||||||
|
map[string]any{"user_id": 2, "dates": []string{"2026-06-01"}})
|
||||||
|
if resp.StatusCode != http.StatusConflict {
|
||||||
|
t.Fatalf("expected 409 without replace, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
|
||||||
|
if got := scheduleHolder(t, s, "2026-06-01"); got != "admin" {
|
||||||
|
t.Errorf("expected the original holder untouched, got %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Replace makes a repeated date idempotent rather than a conflict: the second
|
||||||
|
// pass clears what the first wrote and rewrites it. Worth pinning down, because
|
||||||
|
// the same input without replace is a 409.
|
||||||
|
func TestSchedule_ReplaceCollapsesRepeatedDates(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/schedule",
|
||||||
|
map[string]any{"user_id": 1, "dates": []string{"2026-06-01", "2026-06-01"}, "replace": true})
|
||||||
|
if resp.StatusCode != http.StatusCreated {
|
||||||
|
t.Fatalf("expected a repeated date to be accepted under replace, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
|
||||||
|
var entries []map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/schedule?from=2026-06-01&to=2026-06-01", nil), &entries)
|
||||||
|
if len(entries) != 1 {
|
||||||
|
t.Errorf("expected one entry for the repeated date, got %d", len(entries))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Stats
|
// Stats
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -361,35 +483,29 @@ func TestStats_ByHourReturnsTwentyFourSlots(t *testing.T) {
|
|||||||
// Archive
|
// Archive
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
func TestArchive_RoundTrip(t *testing.T) {
|
// Alert archiving is sweeper-only housekeeping now — nobody archives an alert by
|
||||||
|
// hand — but the list filter it drives is still part of the API.
|
||||||
|
func TestArchive_AlertListFilter(t *testing.T) {
|
||||||
s := newTS(t)
|
s := newTS(t)
|
||||||
|
|
||||||
postWebhook(t, s, []map[string]any{{
|
postWebhook(t, s, []map[string]any{{
|
||||||
"status": "resolved", "fingerprint": "arch1",
|
"status": "resolved", "fingerprint": "arch1",
|
||||||
"labels": map[string]string{"alertname": "Archivable"},
|
"labels": map[string]string{"alertname": "Archivable"},
|
||||||
"annotations": map[string]string{},
|
"annotations": map[string]string{},
|
||||||
"startsAt": "2026-05-20T10:00:00Z", "endsAt": "2026-05-20T11:00:00Z",
|
"startsAt": "2026-05-20T10:00:00Z",
|
||||||
|
"endsAt": "2026-05-20T11:00:00Z",
|
||||||
"generatorURL": "",
|
"generatorURL": "",
|
||||||
}})
|
}})
|
||||||
|
|
||||||
// 1. Alert appears in default list (not archived).
|
// 1. Alert appears in the default list.
|
||||||
var alerts []map[string]any
|
var alerts []map[string]any
|
||||||
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
||||||
if len(alerts) != 1 {
|
if len(alerts) != 1 {
|
||||||
t.Fatalf("expected 1 alert in default list, got %d", len(alerts))
|
t.Fatalf("expected 1 alert in default list, got %d", len(alerts))
|
||||||
}
|
}
|
||||||
id := int(alerts[0]["id"].(float64))
|
|
||||||
|
|
||||||
// 2. Archive it.
|
// 2. Let the sweeper archive it: ends_at is already well past archiveAfter.
|
||||||
resp := s.req(t, http.MethodPost, fmt.Sprintf("/api/alerts/%d/archive", id), nil)
|
api.Sweep(context.Background(), s.db, time.Hour, 6*time.Hour, s.deadman, s.notify)
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
t.Fatalf("archive: expected 200, got %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
var archived map[string]any
|
|
||||||
decode(t, resp, &archived)
|
|
||||||
if archived["archived_at"] == nil {
|
|
||||||
t.Error("expected archived_at to be set in response")
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. Default list excludes it.
|
// 3. Default list excludes it.
|
||||||
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
||||||
@@ -402,18 +518,290 @@ func TestArchive_RoundTrip(t *testing.T) {
|
|||||||
if len(alerts) != 1 {
|
if len(alerts) != 1 {
|
||||||
t.Fatalf("expected 1 archived alert, got %d", len(alerts))
|
t.Fatalf("expected 1 archived alert, got %d", len(alerts))
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// 5. Un-archive.
|
// ---------------------------------------------------------------------------
|
||||||
resp = s.req(t, http.MethodDelete, fmt.Sprintf("/api/alerts/%d/archive", id), nil)
|
// Stale-alert expiry
|
||||||
if resp.StatusCode != http.StatusNoContent {
|
// ---------------------------------------------------------------------------
|
||||||
t.Fatalf("unarchive: expected 204, got %d", resp.StatusCode)
|
|
||||||
|
// noArchive is long enough that archiving never interferes with expiry tests.
|
||||||
|
const noArchive = 365 * 24 * time.Hour
|
||||||
|
|
||||||
|
// zeroTime is Alertmanager's "no end known" sentinel, which stores ends_at NULL.
|
||||||
|
const zeroTime = "0001-01-01T00:00:00Z"
|
||||||
|
|
||||||
|
// postAlert sends a single-alert webhook.
|
||||||
|
func postAlert(t *testing.T, s *ts, fingerprint, status, startsAt, endsAt string) {
|
||||||
|
t.Helper()
|
||||||
|
postWebhook(t, s, []map[string]any{{
|
||||||
|
"status": status,
|
||||||
|
"labels": map[string]string{"alertname": "Stale"},
|
||||||
|
"annotations": map[string]string{},
|
||||||
|
"startsAt": startsAt,
|
||||||
|
"endsAt": endsAt,
|
||||||
|
"generatorURL": "",
|
||||||
|
"fingerprint": fingerprint,
|
||||||
|
}})
|
||||||
|
}
|
||||||
|
|
||||||
|
func sweep(t *testing.T, s *ts, staleAfter time.Duration) {
|
||||||
|
t.Helper()
|
||||||
|
api.Sweep(context.Background(), s.db, noArchive, staleAfter, s.deadman, s.notify)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A firing alert Alertmanager stopped refreshing is resolved via the
|
||||||
|
// received_at heartbeat, even with no ends_at watermark to go on.
|
||||||
|
func TestExpiry_StaleFiringAlert(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postAlert(t, s, "stale1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
||||||
|
|
||||||
|
// Age the last-seen timestamp past the staleness window.
|
||||||
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'stale1'",
|
||||||
|
time.Now().Add(-10*time.Hour).Unix())
|
||||||
|
|
||||||
|
sweep(t, s, 6*time.Hour)
|
||||||
|
|
||||||
|
status, source, _ := s.alertRow(t, "stale1")
|
||||||
|
if status != "resolved" {
|
||||||
|
t.Errorf("expected status resolved, got %q", status)
|
||||||
|
}
|
||||||
|
if source == nil || *source != "expiry" {
|
||||||
|
t.Errorf("expected resolution_source=expiry, got %v", source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A fresh webhook whose ends_at watermark has already passed is expired without
|
||||||
|
// waiting out the full staleness window.
|
||||||
|
func TestExpiry_PastEndsAt(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postAlert(t, s, "stale2", "firing",
|
||||||
|
time.Now().Add(-2*time.Hour).Format(time.RFC3339),
|
||||||
|
time.Now().Add(-30*time.Minute).Format(time.RFC3339))
|
||||||
|
|
||||||
|
sweep(t, s, 6*time.Hour) // received_at is fresh; only ends_at can trigger
|
||||||
|
|
||||||
|
status, source, _ := s.alertRow(t, "stale2")
|
||||||
|
if status != "resolved" {
|
||||||
|
t.Errorf("expected status resolved, got %q", status)
|
||||||
|
}
|
||||||
|
if source == nil || *source != "expiry" {
|
||||||
|
t.Errorf("expected resolution_source=expiry, got %v", source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The regression that matters most: a genuinely firing alert must survive a
|
||||||
|
// sweep untouched.
|
||||||
|
func TestExpiry_LeavesFreshAlertsAlone(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postAlert(t, s, "fresh1", "firing",
|
||||||
|
time.Now().Add(-10*time.Minute).Format(time.RFC3339),
|
||||||
|
time.Now().Add(1*time.Hour).Format(time.RFC3339))
|
||||||
|
|
||||||
|
sweep(t, s, 6*time.Hour)
|
||||||
|
|
||||||
|
status, source, _ := s.alertRow(t, "fresh1")
|
||||||
|
if status != "firing" {
|
||||||
|
t.Errorf("expected fresh alert to stay firing, got %q", status)
|
||||||
|
}
|
||||||
|
if source != nil {
|
||||||
|
t.Errorf("expected no resolution_source, got %q", *source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ends_at only just past must not trip expiry — that grace absorbs clock skew.
|
||||||
|
func TestExpiry_RespectsGraceOnEndsAt(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postAlert(t, s, "grace1", "firing",
|
||||||
|
time.Now().Add(-time.Hour).Format(time.RFC3339),
|
||||||
|
time.Now().Add(-1*time.Minute).Format(time.RFC3339))
|
||||||
|
|
||||||
|
sweep(t, s, 6*time.Hour)
|
||||||
|
|
||||||
|
if status, _, _ := s.alertRow(t, "grace1"); status != "firing" {
|
||||||
|
t.Errorf("expected alert within grace period to stay firing, got %q", status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Webhook resolution bookkeeping
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestWebhook_ResolvedSetsSource(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
||||||
|
postAlert(t, s, "src1", "firing", start, zeroTime)
|
||||||
|
|
||||||
|
if _, source, _ := s.alertRow(t, "src1"); source != nil {
|
||||||
|
t.Errorf("expected firing alert to have no resolution_source, got %q", *source)
|
||||||
|
}
|
||||||
|
|
||||||
|
postAlert(t, s, "src1", "resolved", start, time.Now().Format(time.RFC3339))
|
||||||
|
|
||||||
|
status, source, _ := s.alertRow(t, "src1")
|
||||||
|
if status != "resolved" {
|
||||||
|
t.Errorf("expected status resolved, got %q", status)
|
||||||
|
}
|
||||||
|
if source == nil || *source != "alertmanager" {
|
||||||
|
t.Errorf("expected resolution_source=alertmanager, got %v", source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A re-fire under the same fingerprint must leave the archive and clear the
|
||||||
|
// stale expiry marker, otherwise the alert stays invisible in the default list.
|
||||||
|
func TestWebhook_RefireUnarchivesAndClearsSource(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postAlert(t, s, "refire1", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
||||||
|
|
||||||
|
// Expire it, then archive it.
|
||||||
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'refire1'",
|
||||||
|
time.Now().Add(-10*time.Hour).Unix())
|
||||||
|
sweep(t, s, 6*time.Hour)
|
||||||
|
s.exec(t, "UPDATE alerts SET archived_at = unixepoch() WHERE fingerprint = 'refire1'")
|
||||||
|
|
||||||
|
var alerts []map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
||||||
|
if len(alerts) != 0 {
|
||||||
|
t.Fatalf("expected archived alert to be hidden, got %d", len(alerts))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fires again: a new alert instance, so a newer startsAt.
|
||||||
|
postAlert(t, s, "refire1", "firing", time.Now().Format(time.RFC3339), zeroTime)
|
||||||
|
|
||||||
|
status, source, archivedAt := s.alertRow(t, "refire1")
|
||||||
|
if status != "firing" {
|
||||||
|
t.Errorf("expected status firing after re-fire, got %q", status)
|
||||||
|
}
|
||||||
|
if source != nil {
|
||||||
|
t.Errorf("expected resolution_source cleared on re-fire, got %q", *source)
|
||||||
|
}
|
||||||
|
if archivedAt != nil {
|
||||||
|
t.Errorf("expected archived_at cleared on re-fire, got %d", *archivedAt)
|
||||||
}
|
}
|
||||||
resp.Body.Close()
|
|
||||||
|
|
||||||
// 6. Back in default list.
|
|
||||||
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
||||||
if len(alerts) != 1 {
|
if len(alerts) != 1 {
|
||||||
t.Errorf("expected unarchived alert to reappear, got %d results", len(alerts))
|
t.Errorf("expected re-fired alert back in default list, got %d", len(alerts))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Alertmanager retries failed notifications, so a firing payload for an
|
||||||
|
// already-resolved instance can arrive late. It must not resurrect the alert.
|
||||||
|
func TestWebhook_IgnoresOutOfOrderRetry(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
||||||
|
end := time.Now().Format(time.RFC3339)
|
||||||
|
|
||||||
|
postAlert(t, s, "ooo1", "firing", start, zeroTime)
|
||||||
|
postAlert(t, s, "ooo1", "resolved", start, end)
|
||||||
|
postAlert(t, s, "ooo1", "firing", start, zeroTime) // stale retry, same instance
|
||||||
|
|
||||||
|
status, source, _ := s.alertRow(t, "ooo1")
|
||||||
|
if status != "resolved" {
|
||||||
|
t.Errorf("expected alert to stay resolved after stale retry, got %q", status)
|
||||||
|
}
|
||||||
|
if source == nil || *source != "alertmanager" {
|
||||||
|
t.Errorf("expected resolution_source=alertmanager, got %v", source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An expiry resolve writes ends_at as an upper bound, not an observed end: an
|
||||||
|
// Alertmanager watermark already on the row is preserved, and a row that never
|
||||||
|
// carried one is stamped at sweep time. Clients are told to read it that way —
|
||||||
|
// see "resolution_source says how much to trust ends_at" in the README.
|
||||||
|
func TestExpiry_EndsAtIsUpperBound(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
|
||||||
|
// No watermark: expires on the received_at heartbeat, so the sweeper has
|
||||||
|
// nothing to go on but its own clock.
|
||||||
|
postAlert(t, s, "ub-none", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
||||||
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'ub-none'",
|
||||||
|
time.Now().Add(-10*time.Hour).Unix())
|
||||||
|
|
||||||
|
// Stale watermark: expires on the ends_at branch, and that reported time
|
||||||
|
// must survive the resolve rather than be overwritten with sweep time.
|
||||||
|
watermark := time.Now().Add(-90 * time.Minute).Truncate(time.Second)
|
||||||
|
postAlert(t, s, "ub-mark", "firing",
|
||||||
|
time.Now().Add(-3*time.Hour).Format(time.RFC3339), watermark.Format(time.RFC3339))
|
||||||
|
|
||||||
|
sweep(t, s, 6*time.Hour)
|
||||||
|
|
||||||
|
if _, source, _ := s.alertRow(t, "ub-none"); source == nil || *source != "expiry" {
|
||||||
|
t.Fatalf("expected resolution_source=expiry for heartbeat expiry, got %v", source)
|
||||||
|
}
|
||||||
|
stamped := s.alertEndsAt(t, "ub-none")
|
||||||
|
if stamped == nil {
|
||||||
|
t.Fatal("expected expiry to stamp ends_at when no watermark was known")
|
||||||
|
}
|
||||||
|
if skew := time.Now().Unix() - *stamped; skew < 0 || skew > 5 {
|
||||||
|
t.Errorf("expected stamped ends_at at sweep time, off by %ds", skew)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, source, _ := s.alertRow(t, "ub-mark"); source == nil || *source != "expiry" {
|
||||||
|
t.Fatalf("expected resolution_source=expiry for watermark expiry, got %v", source)
|
||||||
|
}
|
||||||
|
switch kept := s.alertEndsAt(t, "ub-mark"); {
|
||||||
|
case kept == nil:
|
||||||
|
t.Errorf("expected reported watermark %d preserved, got NULL", watermark.Unix())
|
||||||
|
case *kept != watermark.Unix():
|
||||||
|
t.Errorf("expected reported watermark %d preserved, got %d", watermark.Unix(), *kept)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// received_at heartbeat contract
|
||||||
|
//
|
||||||
|
// received_at is documented as a public liveness signal, so these lock the
|
||||||
|
// behaviour clients are told they may rely on. See "received_at is a liveness
|
||||||
|
// heartbeat" in the README and the comment on models.Alert.ReceivedAt.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// The heartbeat itself: an unchanged firing notification — what Alertmanager
|
||||||
|
// re-sends every repeat_interval — must advance received_at, while leaving
|
||||||
|
// starts_at, which identifies the alert instance, untouched.
|
||||||
|
func TestWebhook_ResendBumpsReceivedAt(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
start := time.Now().Add(-24 * time.Hour).Format(time.RFC3339)
|
||||||
|
postAlert(t, s, "beat1", "firing", start, zeroTime)
|
||||||
|
|
||||||
|
startsBefore, _ := s.alertTimes(t, "beat1")
|
||||||
|
|
||||||
|
// received_at has one-second granularity, so back-date it to make the bump
|
||||||
|
// observable instead of sleeping out a second.
|
||||||
|
aged := time.Now().Add(-2 * time.Hour).Unix()
|
||||||
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat1'", aged)
|
||||||
|
|
||||||
|
// Identical re-send: same fingerprint, same startsAt, still firing.
|
||||||
|
postAlert(t, s, "beat1", "firing", start, zeroTime)
|
||||||
|
|
||||||
|
startsAfter, receivedAfter := s.alertTimes(t, "beat1")
|
||||||
|
if receivedAfter <= aged {
|
||||||
|
t.Errorf("expected re-send to advance received_at past %d, got %d", aged, receivedAfter)
|
||||||
|
}
|
||||||
|
if skew := time.Now().Unix() - receivedAfter; skew < 0 || skew > 5 {
|
||||||
|
t.Errorf("expected received_at to track the server clock, off by %ds", skew)
|
||||||
|
}
|
||||||
|
if startsAfter != startsBefore {
|
||||||
|
t.Errorf("expected starts_at unchanged by re-send, got %d want %d", startsAfter, startsBefore)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// received_at tracks accepted payloads, not delivery attempts: a retry
|
||||||
|
// describing an already-resolved instance is discarded, so it must not register
|
||||||
|
// as a heartbeat and revive the alert's apparent liveness.
|
||||||
|
func TestWebhook_DiscardedRetryLeavesReceivedAtAlone(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
start := time.Now().Add(-time.Hour).Format(time.RFC3339)
|
||||||
|
|
||||||
|
postAlert(t, s, "beat2", "firing", start, zeroTime)
|
||||||
|
postAlert(t, s, "beat2", "resolved", start, time.Now().Format(time.RFC3339))
|
||||||
|
|
||||||
|
aged := time.Now().Add(-2 * time.Hour).Unix()
|
||||||
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'beat2'", aged)
|
||||||
|
|
||||||
|
postAlert(t, s, "beat2", "firing", start, zeroTime) // stale retry, discarded
|
||||||
|
|
||||||
|
if _, receivedAfter := s.alertTimes(t, "beat2"); receivedAfter != aged {
|
||||||
|
t.Errorf("expected discarded retry to leave received_at at %d, got %d", aged, receivedAfter)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+223
-20
@@ -4,36 +4,239 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"log"
|
"log"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter time.Duration) {
|
const (
|
||||||
ticker := time.NewTicker(15 * time.Minute)
|
// sweepInterval is how often the background sweeper runs.
|
||||||
|
sweepInterval = 15 * time.Minute
|
||||||
|
|
||||||
|
// expiryGrace absorbs clock skew and notification latency before an alert
|
||||||
|
// whose ends_at watermark has passed is treated as stale.
|
||||||
|
expiryGrace = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// StartArchiver runs the alert sweeper until ctx is cancelled, starting with an
|
||||||
|
// immediate pass so a restart reconciles state right away.
|
||||||
|
func StartArchiver(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration, deadman DeadmanConfig, notify NotifyConfig) {
|
||||||
|
ticker := time.NewTicker(sweepInterval)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
|
||||||
run := func() {
|
Sweep(ctx, db, archiveAfter, staleAfter, deadman, notify)
|
||||||
cutoff := time.Now().Add(-archiveAfter).Unix()
|
|
||||||
res, err := db.ExecContext(ctx,
|
|
||||||
`UPDATE alerts SET archived_at = unixepoch()
|
|
||||||
WHERE status = 'resolved'
|
|
||||||
AND archived_at IS NULL
|
|
||||||
AND COALESCE(ends_at, received_at) < ?`, cutoff)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("archiver: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if n, _ := res.RowsAffected(); n > 0 {
|
|
||||||
log.Printf("archiver: archived %d resolved alert(s)", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
run()
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
run()
|
Sweep(ctx, db, archiveAfter, staleAfter, deadman, notify)
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sweep runs a single pass, in dependency order: reconcile the dead man's
|
||||||
|
// switches, expire stale firing alerts, close the incidents that leaves with
|
||||||
|
// nothing firing, then archive whatever has been settled long enough. Running
|
||||||
|
// them in one pass means an alert can go stale and its incident can close and
|
||||||
|
// archive without waiting three ticks.
|
||||||
|
//
|
||||||
|
// The switches go first because they hand expireStale the alerts it must not
|
||||||
|
// touch: a heartbeat answers to its own, much tighter, timeout, and the generic
|
||||||
|
// staleness rules would otherwise resolve it as 'expiry' long before that.
|
||||||
|
// Exported so tests can drive a pass without waiting on the ticker.
|
||||||
|
func Sweep(ctx context.Context, db *sql.DB, archiveAfter, staleAfter time.Duration, deadman DeadmanConfig, notify NotifyConfig) {
|
||||||
|
heartbeats := sweepDeadman(ctx, db, deadman, notify)
|
||||||
|
expireStale(ctx, db, staleAfter, heartbeats)
|
||||||
|
resolveSettledIncidents(ctx, db)
|
||||||
|
archiveResolved(ctx, db, archiveAfter)
|
||||||
|
archiveResolvedIncidents(ctx, db, archiveAfter)
|
||||||
|
purgeAckTokens(ctx, db)
|
||||||
|
purgeSessions(ctx, db)
|
||||||
|
}
|
||||||
|
|
||||||
|
// expireStale resolves firing alerts that Alertmanager has stopped refreshing.
|
||||||
|
//
|
||||||
|
// A resolved webhook is otherwise the only way out of the firing state, so a
|
||||||
|
// notification that is dropped, silenced, or lost to a restart would pin the
|
||||||
|
// alert as firing forever. Two independent signals mark an alert stale:
|
||||||
|
//
|
||||||
|
// - ends_at, the "valid until" watermark Alertmanager sets on outgoing firing
|
||||||
|
// notifications, has passed (plus expiryGrace for clock skew). Absent on
|
||||||
|
// rows whose payload carried no ends_at, hence the second signal.
|
||||||
|
// - received_at is older than staleAfter. Alertmanager re-sends firing
|
||||||
|
// notifications every repeat_interval, making received_at a liveness
|
||||||
|
// heartbeat — provided staleAfter exceeds that interval.
|
||||||
|
//
|
||||||
|
// Alerts in skip are left alone: they are dead man's switch heartbeats, whose
|
||||||
|
// liveness sweepDeadman has already judged against a timeout of its own.
|
||||||
|
//
|
||||||
|
// The matching rows are collected before the update rather than updated in bulk,
|
||||||
|
// because each one owes its incident a timeline entry.
|
||||||
|
func expireStale(ctx context.Context, db *sql.DB, staleAfter time.Duration, skip map[int64]bool) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
found, err := staleAlertIDs(ctx, db, now, staleAfter)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("sweeper: find stale: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ids := make([]int64, 0, len(found))
|
||||||
|
for _, id := range found {
|
||||||
|
if !skip[id] {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(ids) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
args := make([]any, 0, len(ids)+1)
|
||||||
|
args = append(args, resolutionExpiry)
|
||||||
|
for _, id := range ids {
|
||||||
|
args = append(args, id)
|
||||||
|
}
|
||||||
|
if _, err := db.ExecContext(ctx, `
|
||||||
|
UPDATE alerts
|
||||||
|
SET status = 'resolved',
|
||||||
|
resolution_source = ?,
|
||||||
|
ends_at = COALESCE(ends_at, unixepoch())
|
||||||
|
WHERE id IN (`+placeholders(len(ids))+`)`, args...); err != nil {
|
||||||
|
log.Printf("sweeper: expire stale: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
log.Printf("sweeper: expired %d stale firing alert(s)", len(ids))
|
||||||
|
|
||||||
|
for _, id := range ids {
|
||||||
|
incidentID, err := openIncidentForAlert(ctx, db, id)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("sweeper: incident for alert %d: %v", id, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if incidentID == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
alertID := id
|
||||||
|
if err := logEvent(ctx, db, incidentID, evAlertResolved, nil, &alertID, nil); err != nil {
|
||||||
|
log.Printf("sweeper: log expiry event: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// staleAlertIDs reads the ids in one go and closes the cursor before the caller
|
||||||
|
// writes: the pool is limited to a single connection, so an open read would
|
||||||
|
// block the update behind it.
|
||||||
|
func staleAlertIDs(ctx context.Context, db *sql.DB, now time.Time, staleAfter time.Duration) ([]int64, error) {
|
||||||
|
rows, err := db.QueryContext(ctx, `
|
||||||
|
SELECT id FROM alerts
|
||||||
|
WHERE status = 'firing'
|
||||||
|
AND archived_at IS NULL
|
||||||
|
AND ((ends_at IS NOT NULL AND ends_at < ?) OR received_at < ?)`,
|
||||||
|
now.Add(-expiryGrace).Unix(), now.Add(-staleAfter).Unix())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var ids []int64
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
if err := rows.Scan(&id); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
return ids, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveSettledIncidents closes incidents whose alerts have all stopped firing.
|
||||||
|
// This is the cascade from alerts up to the work item, and it is what turns an
|
||||||
|
// expiry into a closed incident rather than one that sits open forever.
|
||||||
|
func resolveSettledIncidents(ctx context.Context, db *sql.DB) {
|
||||||
|
ids, err := settledIncidentIDs(ctx, db)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("sweeper: find settled incidents: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved := 0
|
||||||
|
for _, id := range ids {
|
||||||
|
ok, err := resolveIfSettled(ctx, db, id)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("sweeper: resolve incident %d: %v", id, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
resolved++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if resolved > 0 {
|
||||||
|
log.Printf("sweeper: resolved %d settled incident(s)", resolved)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func settledIncidentIDs(ctx context.Context, db *sql.DB) ([]int64, error) {
|
||||||
|
rows, err := db.QueryContext(ctx, `
|
||||||
|
SELECT i.id
|
||||||
|
FROM incidents i
|
||||||
|
WHERE i.resolved_at IS NULL
|
||||||
|
AND EXISTS (SELECT 1 FROM incident_alerts ia WHERE ia.incident_id = i.id)
|
||||||
|
AND NOT EXISTS (SELECT 1
|
||||||
|
FROM incident_alerts ia
|
||||||
|
JOIN alerts a ON a.id = ia.alert_id
|
||||||
|
WHERE ia.incident_id = i.id
|
||||||
|
AND a.status = 'firing')`)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var ids []int64
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
if err := rows.Scan(&id); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
return ids, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// archiveResolved hides resolved alerts that have been settled for archiveAfter.
|
||||||
|
func archiveResolved(ctx context.Context, db *sql.DB, archiveAfter time.Duration) {
|
||||||
|
cutoff := time.Now().Add(-archiveAfter).Unix()
|
||||||
|
res, err := db.ExecContext(ctx,
|
||||||
|
`UPDATE alerts SET archived_at = unixepoch()
|
||||||
|
WHERE status = 'resolved'
|
||||||
|
AND archived_at IS NULL
|
||||||
|
AND COALESCE(ends_at, received_at) < ?`, cutoff)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("archiver: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n > 0 {
|
||||||
|
log.Printf("archiver: archived %d resolved alert(s)", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// archiveResolvedIncidents does the same for the work items, on the same clock.
|
||||||
|
func archiveResolvedIncidents(ctx context.Context, db *sql.DB, archiveAfter time.Duration) {
|
||||||
|
cutoff := time.Now().Add(-archiveAfter).Unix()
|
||||||
|
res, err := db.ExecContext(ctx,
|
||||||
|
`UPDATE incidents SET archived_at = unixepoch()
|
||||||
|
WHERE resolved_at IS NOT NULL
|
||||||
|
AND archived_at IS NULL
|
||||||
|
AND resolved_at < ?`, cutoff)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("archiver: incidents: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n > 0 {
|
||||||
|
log.Printf("archiver: archived %d resolved incident(s)", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// placeholders builds "?, ?, …" for an IN clause of n values.
|
||||||
|
func placeholders(n int) string {
|
||||||
|
return strings.TrimSuffix(strings.Repeat("?, ", n), ", ")
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,356 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"log"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// sessionCookie carries a web UI session. It is HttpOnly, so page script
|
||||||
|
// never sees the token; the page learns who it is from GET /api/me.
|
||||||
|
sessionCookie = "terdut_session"
|
||||||
|
|
||||||
|
// sessionTTL is how long a session lives without being used. It slides, so
|
||||||
|
// a phone that opens the UI now and then stays signed in indefinitely.
|
||||||
|
sessionTTL = 30 * 24 * time.Hour
|
||||||
|
|
||||||
|
// sessionTouchEvery bounds how often a request may slide the expiry.
|
||||||
|
sessionTouchEvery = time.Hour
|
||||||
|
|
||||||
|
minPasswordLen = 10
|
||||||
|
// maxPasswordLen is bcrypt's limit; it rejects longer input outright.
|
||||||
|
maxPasswordLen = 72
|
||||||
|
|
||||||
|
loginWindow = 15 * time.Minute
|
||||||
|
loginMaxPerUser = 10
|
||||||
|
loginMaxPerAddr = 30
|
||||||
|
passwordHashCost = bcrypt.DefaultCost
|
||||||
|
)
|
||||||
|
|
||||||
|
// dummyHash is compared against when the username is unknown or has no
|
||||||
|
// password, so a failed login takes as long whichever way it failed.
|
||||||
|
var dummyHash = sync.OnceValue(func() []byte {
|
||||||
|
h, _ := bcrypt.GenerateFromPassword([]byte("terdut-dummy-password"), passwordHashCost)
|
||||||
|
return h
|
||||||
|
})
|
||||||
|
|
||||||
|
// loginLimiter counts failed logins in a fixed window, per username and per
|
||||||
|
// client address. The username limit is what stops guessing one account; the
|
||||||
|
// address limit is looser because every user behind the same gateway or NAT
|
||||||
|
// shares it.
|
||||||
|
type loginLimiter struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
failures map[string]*loginWindowCount
|
||||||
|
}
|
||||||
|
|
||||||
|
type loginWindowCount struct {
|
||||||
|
start time.Time
|
||||||
|
n int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newLoginLimiter() *loginLimiter {
|
||||||
|
return &loginLimiter{failures: map[string]*loginWindowCount{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *loginLimiter) blocked(key string, max int) bool {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
c, ok := l.failures[key]
|
||||||
|
if !ok || time.Since(c.start) > loginWindow {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return c.n >= max
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *loginLimiter) fail(keys ...string) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
now := time.Now()
|
||||||
|
for k, c := range l.failures {
|
||||||
|
if now.Sub(c.start) > loginWindow {
|
||||||
|
delete(l.failures, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, key := range keys {
|
||||||
|
c, ok := l.failures[key]
|
||||||
|
if !ok {
|
||||||
|
c = &loginWindowCount{start: now}
|
||||||
|
l.failures[key] = c
|
||||||
|
}
|
||||||
|
c.n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *loginLimiter) clear(key string) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
delete(l.failures, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// clientAddr is the address a login is counted against. Behind the gateway
|
||||||
|
// RemoteAddr is the gateway itself, so the first X-Forwarded-For hop is used
|
||||||
|
// when present. It can be forged, but only to dodge the address limit; the
|
||||||
|
// per-username limit does not depend on it.
|
||||||
|
func clientAddr(r *http.Request) string {
|
||||||
|
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
|
||||||
|
first, _, _ := strings.Cut(xff, ",")
|
||||||
|
return strings.TrimSpace(first)
|
||||||
|
}
|
||||||
|
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||||
|
if err != nil {
|
||||||
|
return r.RemoteAddr
|
||||||
|
}
|
||||||
|
return host
|
||||||
|
}
|
||||||
|
|
||||||
|
// cookieSecure decides the cookie's Secure flag. TLS terminates at the gateway,
|
||||||
|
// so the server usually sees plain HTTP; the public URL is what says whether
|
||||||
|
// browsers reach it over HTTPS.
|
||||||
|
func cookieSecure(publicURL string, r *http.Request) bool {
|
||||||
|
return strings.HasPrefix(publicURL, "https://") ||
|
||||||
|
r.TLS != nil ||
|
||||||
|
r.Header.Get("X-Forwarded-Proto") == "https"
|
||||||
|
}
|
||||||
|
|
||||||
|
// validatePassword returns a message for the client, or "" when acceptable.
|
||||||
|
func validatePassword(pw string) string {
|
||||||
|
switch {
|
||||||
|
case len(pw) < minPasswordLen:
|
||||||
|
return "password must be at least " + strconv.Itoa(minPasswordLen) + " characters"
|
||||||
|
case len(pw) > maxPasswordLen:
|
||||||
|
return "password must be at most " + strconv.Itoa(maxPasswordLen) + " bytes"
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func hashPassword(pw string) (string, error) {
|
||||||
|
h, err := bcrypt.GenerateFromPassword([]byte(pw), passwordHashCost)
|
||||||
|
return string(h), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleLogin exchanges a username and password for a session cookie.
|
||||||
|
func handleLogin(db *sql.DB, limiter *loginLimiter, publicURL string) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var req struct {
|
||||||
|
Username string `json:"username"`
|
||||||
|
Password string `json:"password"`
|
||||||
|
}
|
||||||
|
if err := decodeJSON(r, &req); err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
username := strings.TrimSpace(req.Username)
|
||||||
|
userKey := "user:" + strings.ToLower(username)
|
||||||
|
addrKey := "addr:" + clientAddr(r)
|
||||||
|
|
||||||
|
if limiter.blocked(userKey, loginMaxPerUser) || limiter.blocked(addrKey, loginMaxPerAddr) {
|
||||||
|
w.Header().Set("Retry-After", strconv.Itoa(int(loginWindow.Seconds())))
|
||||||
|
respond(w, http.StatusTooManyRequests, errResp("too many failed attempts, try again later"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var userID int64
|
||||||
|
var hash sql.NullString
|
||||||
|
err := db.QueryRowContext(r.Context(),
|
||||||
|
"SELECT id, password_hash FROM users WHERE username = ?", username,
|
||||||
|
).Scan(&userID, &hash)
|
||||||
|
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
stored := dummyHash()
|
||||||
|
if hash.Valid {
|
||||||
|
stored = []byte(hash.String)
|
||||||
|
}
|
||||||
|
match := bcrypt.CompareHashAndPassword(stored, []byte(req.Password)) == nil
|
||||||
|
if !match || !hash.Valid {
|
||||||
|
limiter.fail(userKey, addrKey)
|
||||||
|
respond(w, http.StatusUnauthorized, errResp("invalid username or password"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
limiter.clear(userKey)
|
||||||
|
|
||||||
|
raw, tokenHash, err := randomToken()
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
if _, err := db.ExecContext(r.Context(), `
|
||||||
|
INSERT INTO sessions (token_hash, user_id, created_at, last_seen_at, expires_at, user_agent)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||||
|
tokenHash, userID, now.Unix(), now.Unix(), now.Add(sessionTTL).Unix(), r.UserAgent()); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
http.SetCookie(w, &http.Cookie{
|
||||||
|
Name: sessionCookie,
|
||||||
|
Value: raw,
|
||||||
|
Path: "/",
|
||||||
|
MaxAge: int(sessionTTL.Seconds()),
|
||||||
|
HttpOnly: true,
|
||||||
|
Secure: cookieSecure(publicURL, r),
|
||||||
|
SameSite: http.SameSiteLaxMode,
|
||||||
|
})
|
||||||
|
|
||||||
|
user, err := fetchUser(r.Context(), db, userID)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respond(w, http.StatusOK, meResponse{User: user, HasPassword: true})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleLogout ends the browser's session. It sits outside AuthMiddleware so
|
||||||
|
// that a browser holding an already-expired cookie can still clear it.
|
||||||
|
func handleLogout(db *sql.DB, publicURL string) http.HandlerFunc {
|
||||||
|
crossOrigin := http.NewCrossOriginProtection()
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if err := crossOrigin.Check(r); err != nil {
|
||||||
|
respond(w, http.StatusForbidden, errResp("cross-origin request rejected"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if c, err := r.Cookie(sessionCookie); err == nil && c.Value != "" {
|
||||||
|
db.ExecContext(r.Context(), "DELETE FROM sessions WHERE token_hash = ?", hashToken(c.Value))
|
||||||
|
}
|
||||||
|
http.SetCookie(w, &http.Cookie{
|
||||||
|
Name: sessionCookie,
|
||||||
|
Value: "",
|
||||||
|
Path: "/",
|
||||||
|
MaxAge: -1,
|
||||||
|
HttpOnly: true,
|
||||||
|
Secure: cookieSecure(publicURL, r),
|
||||||
|
SameSite: http.SameSiteLaxMode,
|
||||||
|
})
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type meResponse struct {
|
||||||
|
User any `json:"user"`
|
||||||
|
HasPassword bool `json:"has_password"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleMe says who the caller is. The web UI calls it on load to decide
|
||||||
|
// between the login form and the app, since it cannot read its own cookie.
|
||||||
|
func handleMe(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
caller, _ := userFromContext(r.Context())
|
||||||
|
user, err := fetchUser(r.Context(), db, caller.ID)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var hash sql.NullString
|
||||||
|
db.QueryRowContext(r.Context(),
|
||||||
|
"SELECT password_hash FROM users WHERE id = ?", caller.ID).Scan(&hash)
|
||||||
|
respond(w, http.StatusOK, meResponse{User: user, HasPassword: hash.Valid})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleSetPassword sets a user's web UI password.
|
||||||
|
//
|
||||||
|
// Changing your own password takes the current one, when there is one, so an
|
||||||
|
// unattended signed-in browser cannot be used to take the account over. Setting
|
||||||
|
// somebody else's is how an admin gives a user their first password, and like
|
||||||
|
// the other user endpoints it is open to any authenticated caller.
|
||||||
|
//
|
||||||
|
// Every other session of the target is ended: a password change is what you
|
||||||
|
// do when you think someone else is signed in.
|
||||||
|
func handleSetPassword(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid user id"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req struct {
|
||||||
|
Password string `json:"password"`
|
||||||
|
CurrentPassword string `json:"current_password"`
|
||||||
|
}
|
||||||
|
if err := decodeJSON(r, &req); err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if msg := validatePassword(req.Password); msg != "" {
|
||||||
|
respond(w, http.StatusBadRequest, errResp(msg))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var existing sql.NullString
|
||||||
|
err = db.QueryRowContext(r.Context(),
|
||||||
|
"SELECT password_hash FROM users WHERE id = ?", id).Scan(&existing)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
respond(w, http.StatusNotFound, errResp("user not found"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
caller, _ := userFromContext(r.Context())
|
||||||
|
if caller.ID == id && existing.Valid &&
|
||||||
|
bcrypt.CompareHashAndPassword([]byte(existing.String), []byte(req.CurrentPassword)) != nil {
|
||||||
|
respond(w, http.StatusForbidden, errResp("current password is incorrect"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
hash, err := hashPassword(req.Password)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
tx, err := db.BeginTx(r.Context(), nil)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
|
||||||
|
if _, err := tx.ExecContext(r.Context(),
|
||||||
|
"UPDATE users SET password_hash = ? WHERE id = ?", hash, id); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
keep, _ := sessionFromContext(r.Context()) // zero when changed with an API key
|
||||||
|
if _, err := tx.ExecContext(r.Context(),
|
||||||
|
"DELETE FROM sessions WHERE user_id = ? AND id != ?", id, keep); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// purgeSessions deletes sessions that have expired, from the sweeper.
|
||||||
|
func purgeSessions(ctx context.Context, db *sql.DB) {
|
||||||
|
res, err := db.ExecContext(ctx,
|
||||||
|
"DELETE FROM sessions WHERE expires_at < ?", time.Now().Unix())
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("sweeper: purge sessions: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n > 0 {
|
||||||
|
log.Printf("sweeper: purged %d expired session(s)", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,360 @@
|
|||||||
|
package api_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/cookiejar"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/api"
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
const adminPassword = "correct horse battery"
|
||||||
|
|
||||||
|
// browser is an HTTP client with its own cookie jar, standing in for one
|
||||||
|
// signed-in browser.
|
||||||
|
type browser struct {
|
||||||
|
*http.Client
|
||||||
|
base string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBrowser(t *testing.T, base string) *browser {
|
||||||
|
t.Helper()
|
||||||
|
jar, _ := cookiejar.New(nil)
|
||||||
|
return &browser{Client: &http.Client{Jar: jar}, base: base}
|
||||||
|
}
|
||||||
|
|
||||||
|
// do sends a request the way the web UI's own fetch would: same-origin, with
|
||||||
|
// the cookie from the jar.
|
||||||
|
func (b *browser) do(t *testing.T, method, path string, body any, header ...string) *http.Response {
|
||||||
|
t.Helper()
|
||||||
|
var r io.Reader
|
||||||
|
if body != nil {
|
||||||
|
data, _ := json.Marshal(body)
|
||||||
|
r = bytes.NewReader(data)
|
||||||
|
}
|
||||||
|
req, _ := http.NewRequest(method, b.base+path, r)
|
||||||
|
if body != nil {
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
}
|
||||||
|
req.Header.Set("Sec-Fetch-Site", "same-origin")
|
||||||
|
for i := 0; i+1 < len(header); i += 2 {
|
||||||
|
req.Header.Set(header[i], header[i+1])
|
||||||
|
}
|
||||||
|
resp, err := b.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s %s: %v", method, path, err)
|
||||||
|
}
|
||||||
|
return resp
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *browser) login(t *testing.T, username, password string) *http.Response {
|
||||||
|
t.Helper()
|
||||||
|
return b.do(t, http.MethodPost, "/api/login", map[string]string{"username": username, "password": password})
|
||||||
|
}
|
||||||
|
|
||||||
|
// setAdminPassword gives the bootstrapped admin a password over its API key.
|
||||||
|
func setAdminPassword(t *testing.T, s *ts) {
|
||||||
|
t.Helper()
|
||||||
|
resp := s.req(t, http.MethodPut, "/api/users/1/password", map[string]string{"password": adminPassword})
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusNoContent {
|
||||||
|
t.Fatalf("set password: %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func signedIn(t *testing.T, s *ts) *browser {
|
||||||
|
t.Helper()
|
||||||
|
setAdminPassword(t, s)
|
||||||
|
b := newBrowser(t, s.URL)
|
||||||
|
resp := b.login(t, "admin", adminPassword)
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("login: %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func status(t *testing.T, resp *http.Response) int {
|
||||||
|
t.Helper()
|
||||||
|
resp.Body.Close()
|
||||||
|
return resp.StatusCode
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogin_SetsSessionCookie(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
setAdminPassword(t, s)
|
||||||
|
b := newBrowser(t, s.URL)
|
||||||
|
|
||||||
|
resp := b.login(t, "admin", adminPassword)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("login: %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
var cookie *http.Cookie
|
||||||
|
for _, c := range resp.Cookies() {
|
||||||
|
if c.Name == "terdut_session" {
|
||||||
|
cookie = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cookie == nil || !cookie.HttpOnly || cookie.SameSite != http.SameSiteLaxMode {
|
||||||
|
t.Fatalf("expected an HttpOnly, SameSite=Lax session cookie, got %+v", cookie)
|
||||||
|
}
|
||||||
|
if cookie.Secure {
|
||||||
|
t.Error("cookie is Secure on a plain-HTTP server with no https public URL")
|
||||||
|
}
|
||||||
|
var me struct {
|
||||||
|
User struct {
|
||||||
|
Username string `json:"username"`
|
||||||
|
} `json:"user"`
|
||||||
|
HasPassword bool `json:"has_password"`
|
||||||
|
}
|
||||||
|
decode(t, resp, &me)
|
||||||
|
if me.User.Username != "admin" || !me.HasPassword {
|
||||||
|
t.Errorf("unexpected login response %+v", me)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogin_CookieAuthenticatesAPI(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
b := signedIn(t, s)
|
||||||
|
|
||||||
|
if code := status(t, b.do(t, http.MethodGet, "/api/incidents", nil)); code != http.StatusOK {
|
||||||
|
t.Errorf("GET /api/incidents with cookie: %d", code)
|
||||||
|
}
|
||||||
|
resp := b.do(t, http.MethodGet, "/api/me", nil)
|
||||||
|
var me struct {
|
||||||
|
User struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
} `json:"user"`
|
||||||
|
}
|
||||||
|
decode(t, resp, &me)
|
||||||
|
if me.User.ID != 1 {
|
||||||
|
t.Errorf("/api/me returned user %d", me.User.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogin_SecureCookieBehindHTTPSPublicURL(t *testing.T) {
|
||||||
|
s := newTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
setAdminPassword(t, s)
|
||||||
|
resp := newBrowser(t, s.URL).login(t, "admin", adminPassword)
|
||||||
|
resp.Body.Close()
|
||||||
|
for _, c := range resp.Cookies() {
|
||||||
|
if c.Name == "terdut_session" && !c.Secure {
|
||||||
|
t.Error("cookie should be Secure when the public URL is https")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogin_WrongPasswordAndUnknownUser(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
setAdminPassword(t, s)
|
||||||
|
b := newBrowser(t, s.URL)
|
||||||
|
|
||||||
|
if code := status(t, b.login(t, "admin", "not the password")); code != http.StatusUnauthorized {
|
||||||
|
t.Errorf("wrong password: %d", code)
|
||||||
|
}
|
||||||
|
if code := status(t, b.login(t, "nobody", adminPassword)); code != http.StatusUnauthorized {
|
||||||
|
t.Errorf("unknown user: %d", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogin_UserWithoutPasswordCannotSignIn(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
b := newBrowser(t, s.URL)
|
||||||
|
// The empty password must not match a user that has none.
|
||||||
|
if code := status(t, b.login(t, "admin", "")); code != http.StatusUnauthorized {
|
||||||
|
t.Errorf("login without a password set: %d", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogin_RateLimitedPerUsername(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
setAdminPassword(t, s)
|
||||||
|
b := newBrowser(t, s.URL)
|
||||||
|
|
||||||
|
for i := range 10 {
|
||||||
|
if code := status(t, b.login(t, "admin", "wrong")); code != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("attempt %d: %d", i+1, code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Even the right password is refused once the limit is reached.
|
||||||
|
resp := b.login(t, "admin", adminPassword)
|
||||||
|
if resp.StatusCode != http.StatusTooManyRequests {
|
||||||
|
t.Fatalf("expected 429, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
if resp.Header.Get("Retry-After") == "" {
|
||||||
|
t.Error("429 without Retry-After")
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSession_CrossOriginWriteRejected(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
b := signedIn(t, s)
|
||||||
|
|
||||||
|
code := status(t, b.do(t, http.MethodPost, "/api/incidents/999/acknowledge", nil,
|
||||||
|
"Sec-Fetch-Site", "cross-site", "Origin", "https://evil.example"))
|
||||||
|
if code != http.StatusForbidden {
|
||||||
|
t.Errorf("cross-origin POST with cookie: %d, want 403", code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same request from the page itself gets through to the handler.
|
||||||
|
code = status(t, b.do(t, http.MethodPost, "/api/incidents/999/acknowledge", nil))
|
||||||
|
if code != http.StatusNotFound {
|
||||||
|
t.Errorf("same-origin POST with cookie: %d, want 404 from the handler", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSession_BearerIgnoresOriginChecks(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
req, _ := http.NewRequest(http.MethodPost, s.URL+"/api/incidents/999/acknowledge", nil)
|
||||||
|
req.Header.Set("Authorization", "Bearer "+s.key)
|
||||||
|
req.Header.Set("Sec-Fetch-Site", "cross-site")
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if code := status(t, resp); code != http.StatusNotFound {
|
||||||
|
t.Errorf("Bearer request: %d, want 404 from the handler", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogout_EndsSession(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
b := signedIn(t, s)
|
||||||
|
|
||||||
|
if code := status(t, b.do(t, http.MethodPost, "/api/logout", nil)); code != http.StatusNoContent {
|
||||||
|
t.Fatalf("logout: %d", code)
|
||||||
|
}
|
||||||
|
if code := status(t, b.do(t, http.MethodGet, "/api/me", nil)); code != http.StatusUnauthorized {
|
||||||
|
t.Errorf("after logout: %d", code)
|
||||||
|
}
|
||||||
|
var n int
|
||||||
|
s.db.QueryRow("SELECT COUNT(*) FROM sessions").Scan(&n)
|
||||||
|
if n != 0 {
|
||||||
|
t.Errorf("%d session(s) left after logout", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSession_ExpiredIsRejected(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
b := signedIn(t, s)
|
||||||
|
s.exec(t, "UPDATE sessions SET expires_at = 1")
|
||||||
|
|
||||||
|
if code := status(t, b.do(t, http.MethodGet, "/api/me", nil)); code != http.StatusUnauthorized {
|
||||||
|
t.Errorf("expired session: %d", code)
|
||||||
|
}
|
||||||
|
api.Sweep(t.Context(), s.db, 0, 0, api.DeadmanConfig{}, api.NotifyConfig{})
|
||||||
|
var n int
|
||||||
|
s.db.QueryRow("SELECT COUNT(*) FROM sessions").Scan(&n)
|
||||||
|
if n != 0 {
|
||||||
|
t.Errorf("sweep left %d expired session(s)", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetPassword_OwnNeedsCurrent(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
b := signedIn(t, s)
|
||||||
|
|
||||||
|
code := status(t, b.do(t, http.MethodPut, "/api/users/1/password",
|
||||||
|
map[string]string{"password": "a brand new secret", "current_password": "wrong"}))
|
||||||
|
if code != http.StatusForbidden {
|
||||||
|
t.Errorf("wrong current password: %d", code)
|
||||||
|
}
|
||||||
|
code = status(t, b.do(t, http.MethodPut, "/api/users/1/password",
|
||||||
|
map[string]string{"password": "short", "current_password": adminPassword}))
|
||||||
|
if code != http.StatusBadRequest {
|
||||||
|
t.Errorf("too-short password: %d", code)
|
||||||
|
}
|
||||||
|
code = status(t, b.do(t, http.MethodPut, "/api/users/1/password",
|
||||||
|
map[string]string{"password": "a brand new secret", "current_password": adminPassword}))
|
||||||
|
if code != http.StatusNoContent {
|
||||||
|
t.Fatalf("change password: %d", code)
|
||||||
|
}
|
||||||
|
if code := status(t, newBrowser(t, s.URL).login(t, "admin", "a brand new secret")); code != http.StatusOK {
|
||||||
|
t.Errorf("login with the new password: %d", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetPassword_EndsOtherSessionsButNotThisOne(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
phone := signedIn(t, s)
|
||||||
|
laptop := newBrowser(t, s.URL)
|
||||||
|
status(t, laptop.login(t, "admin", adminPassword))
|
||||||
|
|
||||||
|
code := status(t, phone.do(t, http.MethodPut, "/api/users/1/password",
|
||||||
|
map[string]string{"password": "a brand new secret", "current_password": adminPassword}))
|
||||||
|
if code != http.StatusNoContent {
|
||||||
|
t.Fatalf("change password: %d", code)
|
||||||
|
}
|
||||||
|
if code := status(t, phone.do(t, http.MethodGet, "/api/me", nil)); code != http.StatusOK {
|
||||||
|
t.Errorf("the session that changed the password: %d", code)
|
||||||
|
}
|
||||||
|
if code := status(t, laptop.do(t, http.MethodGet, "/api/me", nil)); code != http.StatusUnauthorized {
|
||||||
|
t.Errorf("the other session: %d", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBootstrap_WithPassword(t *testing.T) {
|
||||||
|
database, err := db.Open(":memory:")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := db.Migrate(database); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srv := httptest.NewServer(api.NewRouter(database, api.NotifyConfig{}, api.DeadmanConfig{}))
|
||||||
|
t.Cleanup(func() { srv.Close(); database.Close() })
|
||||||
|
|
||||||
|
body := `{"username":"admin","email":"a@test.com","password":"` + adminPassword + `"}`
|
||||||
|
resp, err := http.Post(srv.URL+"/api/bootstrap", "application/json", strings.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if code := status(t, resp); code != http.StatusCreated {
|
||||||
|
t.Fatalf("bootstrap: %d", code)
|
||||||
|
}
|
||||||
|
if code := status(t, newBrowser(t, srv.URL).login(t, "admin", adminPassword)); code != http.StatusOK {
|
||||||
|
t.Errorf("login after bootstrap: %d", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRouter_UnknownAPIPathIsJSON404(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
resp, err := http.Get(s.URL + "/api/nope")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusNotFound || !strings.HasPrefix(resp.Header.Get("Content-Type"), "application/json") {
|
||||||
|
t.Errorf("GET /api/nope: %d %s", resp.StatusCode, resp.Header.Get("Content-Type"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRouter_DeepLinkServesWebUI(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
resp, err := http.Get(s.URL + "/incidents/1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK || !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
|
||||||
|
t.Errorf("GET /incidents/1: %d %s", resp.StatusCode, resp.Header.Get("Content-Type"))
|
||||||
|
}
|
||||||
|
if resp.Header.Get("Content-Security-Policy") == "" {
|
||||||
|
t.Error("web UI served without a CSP")
|
||||||
|
}
|
||||||
|
|
||||||
|
resp2, err := http.Get(s.URL + "/js/missing.js")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if code := status(t, resp2); code != http.StatusNotFound {
|
||||||
|
t.Errorf("missing asset: %d", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,131 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
|
||||||
"github.com/yeniklas/terdut-server/internal/models"
|
|
||||||
)
|
|
||||||
|
|
||||||
func handleListComments(db *sql.DB) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
alertID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusBadRequest, errResp("invalid alert id"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify the alert exists.
|
|
||||||
var exists int
|
|
||||||
if err := db.QueryRowContext(r.Context(), "SELECT 1 FROM alerts WHERE id = ?", alertID).Scan(&exists); err != nil {
|
|
||||||
respond(w, http.StatusNotFound, errResp("alert not found"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rows, err := db.QueryContext(r.Context(), `
|
|
||||||
SELECT c.id, c.alert_id, c.user_id, u.username, c.content, c.created_at
|
|
||||||
FROM alert_comments c
|
|
||||||
JOIN users u ON u.id = c.user_id
|
|
||||||
WHERE c.alert_id = ?
|
|
||||||
ORDER BY c.created_at ASC`, alertID)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
comments := []models.Comment{}
|
|
||||||
for rows.Next() {
|
|
||||||
var c models.Comment
|
|
||||||
var ts int64
|
|
||||||
if err := rows.Scan(&c.ID, &c.AlertID, &c.UserID, &c.Username, &c.Content, &ts); err != nil {
|
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
c.CreatedAt = time.Unix(ts, 0).UTC()
|
|
||||||
comments = append(comments, c)
|
|
||||||
}
|
|
||||||
respond(w, http.StatusOK, comments)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func handleCreateComment(db *sql.DB) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
alertID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusBadRequest, errResp("invalid alert id"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var req struct {
|
|
||||||
Content string `json:"content"`
|
|
||||||
}
|
|
||||||
if err := decodeJSON(r, &req); err != nil {
|
|
||||||
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if req.Content == "" {
|
|
||||||
respond(w, http.StatusBadRequest, errResp("content is required"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify the alert exists.
|
|
||||||
var exists int
|
|
||||||
if err := db.QueryRowContext(r.Context(), "SELECT 1 FROM alerts WHERE id = ?", alertID).Scan(&exists); err != nil {
|
|
||||||
respond(w, http.StatusNotFound, errResp("alert not found"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
user, _ := userFromContext(r.Context())
|
|
||||||
res, err := db.ExecContext(r.Context(),
|
|
||||||
"INSERT INTO alert_comments (alert_id, user_id, content) VALUES (?, ?, ?)",
|
|
||||||
alertID, user.ID, req.Content)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
commentID, _ := res.LastInsertId()
|
|
||||||
|
|
||||||
comment := models.Comment{
|
|
||||||
ID: commentID,
|
|
||||||
AlertID: alertID,
|
|
||||||
UserID: user.ID,
|
|
||||||
Username: user.Username,
|
|
||||||
Content: req.Content,
|
|
||||||
CreatedAt: time.Now().UTC(),
|
|
||||||
}
|
|
||||||
respond(w, http.StatusCreated, comment)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func handleDeleteComment(db *sql.DB) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
alertID, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusBadRequest, errResp("invalid alert id"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
commentID, err := strconv.ParseInt(chi.URLParam(r, "commentID"), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusBadRequest, errResp("invalid comment id"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
user, _ := userFromContext(r.Context())
|
|
||||||
res, err := db.ExecContext(r.Context(),
|
|
||||||
"DELETE FROM alert_comments WHERE id = ? AND alert_id = ? AND user_id = ?",
|
|
||||||
commentID, alertID, user.ID)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if n, _ := res.RowsAffected(); n == 0 {
|
|
||||||
respond(w, http.StatusNotFound, errResp("comment not found"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,378 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"log"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// deadmanGroupPrefix namespaces the incidents this file opens. Alertmanager
|
||||||
|
// group keys always contain braces, so this can never collide with one, and the
|
||||||
|
// partial unique index on open group_key (see 008_incidents.sql) gives one open
|
||||||
|
// incident per switch for free.
|
||||||
|
const deadmanGroupPrefix = "deadman:"
|
||||||
|
|
||||||
|
// DeadmanMatcher selects the alerts that are heartbeats rather than problems.
|
||||||
|
// Every condition has to match, and Name — the alertname label — is mandatory:
|
||||||
|
// it is what lets the sweeper find candidate rows through alerts_name_idx
|
||||||
|
// instead of JSON-extracting labels from every row in the table.
|
||||||
|
type DeadmanMatcher struct {
|
||||||
|
Name string
|
||||||
|
Labels map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// String renders the matcher the way it was configured, which is also how it
|
||||||
|
// reads in an incident title.
|
||||||
|
func (m DeadmanMatcher) String() string {
|
||||||
|
if len(m.Labels) == 0 {
|
||||||
|
return m.Name
|
||||||
|
}
|
||||||
|
parts := make([]string, 0, len(m.Labels))
|
||||||
|
for k, v := range m.Labels {
|
||||||
|
parts = append(parts, k+"="+v)
|
||||||
|
}
|
||||||
|
sort.Strings(parts)
|
||||||
|
return m.Name + " (" + strings.Join(parts, ", ") + ")"
|
||||||
|
}
|
||||||
|
|
||||||
|
// matches reports whether an alert's labels satisfy every condition.
|
||||||
|
func (m DeadmanMatcher) matches(labels map[string]string) bool {
|
||||||
|
if labels["alertname"] != m.Name {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for k, v := range m.Labels {
|
||||||
|
if labels[k] != v {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeadmanConfig inverts the handling of the alerts it matches: receiving one
|
||||||
|
// opens nothing, and the absence of one opens an incident.
|
||||||
|
//
|
||||||
|
// The unit of monitoring is the fingerprint, not the matcher — two clusters
|
||||||
|
// sending the same heartbeat alertname are two independent switches, so one
|
||||||
|
// healthy cluster cannot mask a dead one.
|
||||||
|
type DeadmanConfig struct {
|
||||||
|
Matchers []DeadmanMatcher
|
||||||
|
|
||||||
|
// Timeout is how long a matched alert may go without a refreshing webhook
|
||||||
|
// before it is declared dead. It must be shorter than Alertmanager's
|
||||||
|
// repeat_interval for the heartbeat's route, which is what refreshes it.
|
||||||
|
// Zero disables dead man's switch handling entirely.
|
||||||
|
Timeout time.Duration
|
||||||
|
|
||||||
|
// Severity is the severity every dead man's switch incident opens at. These
|
||||||
|
// incidents have no member alerts to derive one from, and the heartbeat's
|
||||||
|
// own severity label is meaningless — Watchdog ships as "none".
|
||||||
|
Severity string
|
||||||
|
}
|
||||||
|
|
||||||
|
// enabled reports whether there is anything to watch.
|
||||||
|
func (c DeadmanConfig) enabled() bool { return c.Timeout > 0 && len(c.Matchers) > 0 }
|
||||||
|
|
||||||
|
// match returns the first matcher an alert satisfies.
|
||||||
|
func (c DeadmanConfig) match(labels map[string]string) (DeadmanMatcher, bool) {
|
||||||
|
if !c.enabled() {
|
||||||
|
return DeadmanMatcher{}, false
|
||||||
|
}
|
||||||
|
for _, m := range c.Matchers {
|
||||||
|
if m.matches(labels) {
|
||||||
|
return m, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return DeadmanMatcher{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// isDeadman is match without the matcher, for the ingest path.
|
||||||
|
func (c DeadmanConfig) isDeadman(labels map[string]string) bool {
|
||||||
|
_, ok := c.match(labels)
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// names lists the distinct alertnames worth loading from the database.
|
||||||
|
func (c DeadmanConfig) names() []string {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
out := make([]string, 0, len(c.Matchers))
|
||||||
|
for _, m := range c.Matchers {
|
||||||
|
if !seen[m.Name] {
|
||||||
|
seen[m.Name] = true
|
||||||
|
out = append(out, m.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseDeadmanConfig reads the matcher list from its configured form:
|
||||||
|
// ";" separates matchers, "," separates the conditions within one, and "=" is
|
||||||
|
// exact label equality — `alertname=Watchdog,cluster=prod; alertname=Heartbeat`.
|
||||||
|
//
|
||||||
|
// A malformed or alertname-less entry is dropped rather than fatal, following
|
||||||
|
// config.duration's rule that one bad tuning knob should not take the server
|
||||||
|
// down. Silence would be worse here than elsewhere, though — a typo that
|
||||||
|
// disarms the switch is exactly the failure this feature exists to catch — so
|
||||||
|
// the matchers that survived are logged.
|
||||||
|
func ParseDeadmanConfig(matchers string, timeout time.Duration, severity string) DeadmanConfig {
|
||||||
|
cfg := DeadmanConfig{Timeout: timeout, Severity: severity}
|
||||||
|
|
||||||
|
for _, entry := range strings.Split(matchers, ";") {
|
||||||
|
entry = strings.TrimSpace(entry)
|
||||||
|
if entry == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
m := DeadmanMatcher{Labels: map[string]string{}}
|
||||||
|
malformed := false
|
||||||
|
for _, cond := range strings.Split(entry, ",") {
|
||||||
|
k, v, ok := strings.Cut(cond, "=")
|
||||||
|
k, v = strings.TrimSpace(k), strings.TrimSpace(v)
|
||||||
|
if !ok || k == "" || v == "" {
|
||||||
|
log.Printf("deadman: ignoring matcher %q: %q is not label=value", entry, strings.TrimSpace(cond))
|
||||||
|
malformed = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if k == "alertname" {
|
||||||
|
m.Name = v
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m.Labels[k] = v
|
||||||
|
}
|
||||||
|
if malformed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m.Name == "" {
|
||||||
|
log.Printf("deadman: ignoring matcher %q: no alertname condition", entry)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cfg.Matchers = append(cfg.Matchers, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case timeout <= 0:
|
||||||
|
log.Print("deadman: disabled (timeout is zero)")
|
||||||
|
case len(cfg.Matchers) == 0:
|
||||||
|
log.Print("deadman: disabled (no usable matchers)")
|
||||||
|
default:
|
||||||
|
rendered := make([]string, 0, len(cfg.Matchers))
|
||||||
|
for _, m := range cfg.Matchers {
|
||||||
|
rendered = append(rendered, m.String())
|
||||||
|
}
|
||||||
|
log.Printf("deadman: watching %s, timeout %s, severity %s",
|
||||||
|
strings.Join(rendered, "; "), timeout, severity)
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
// deadmanAlert is one switch: the alert row carrying its last heartbeat.
|
||||||
|
type deadmanAlert struct {
|
||||||
|
id int64
|
||||||
|
fingerprint string
|
||||||
|
labels map[string]string
|
||||||
|
matcher DeadmanMatcher
|
||||||
|
resolved bool
|
||||||
|
receivedAt int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// groupKey is the switch's identity as an incident. Per fingerprint, so each
|
||||||
|
// source is tracked on its own.
|
||||||
|
func (a deadmanAlert) groupKey() string { return deadmanGroupPrefix + a.fingerprint }
|
||||||
|
|
||||||
|
// sweepDeadman is the whole point of the feature: it opens an incident for every
|
||||||
|
// switch that has stopped chirping, and closes one whose switch came back.
|
||||||
|
//
|
||||||
|
// It returns the ids of the alerts it owns, because the generic staleness
|
||||||
|
// expiry must leave them alone — staleAfter and ends_at would otherwise resolve
|
||||||
|
// a heartbeat long before its own, much tighter, timeout ever fired.
|
||||||
|
func sweepDeadman(ctx context.Context, db *sql.DB, cfg DeadmanConfig, notify NotifyConfig) map[int64]bool {
|
||||||
|
owned := map[int64]bool{}
|
||||||
|
if !cfg.enabled() {
|
||||||
|
return owned
|
||||||
|
}
|
||||||
|
|
||||||
|
switches, err := deadmanAlerts(ctx, db, cfg)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("deadman: load switches: %v", err)
|
||||||
|
return owned
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
cutoff := now.Add(-cfg.Timeout).Unix()
|
||||||
|
|
||||||
|
for _, sw := range switches {
|
||||||
|
owned[sw.id] = true
|
||||||
|
|
||||||
|
// An explicit resolved from Alertmanager is a stronger death signal than
|
||||||
|
// mere absence: the sender is telling us the heartbeat stopped, so there
|
||||||
|
// is nothing left to wait out.
|
||||||
|
if sw.resolved || sw.receivedAt < cutoff {
|
||||||
|
if err := deadmanDied(ctx, db, cfg, notify, sw, now); err != nil {
|
||||||
|
log.Printf("deadman: open incident for %s: %v", sw.matcher.Name, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := deadmanRecovered(ctx, db, sw); err != nil {
|
||||||
|
log.Printf("deadman: resolve incident for %s: %v", sw.matcher.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return owned
|
||||||
|
}
|
||||||
|
|
||||||
|
// deadmanAlerts loads every alert row that a matcher claims. The candidate query
|
||||||
|
// is narrowed by alertname so it rides alerts_name_idx; the rest of the matching
|
||||||
|
// happens in Go, which keeps one implementation of the rules. The rows are read
|
||||||
|
// in full before the caller writes, because the pool holds a single connection.
|
||||||
|
func deadmanAlerts(ctx context.Context, db *sql.DB, cfg DeadmanConfig) ([]deadmanAlert, error) {
|
||||||
|
names := cfg.names()
|
||||||
|
args := make([]any, 0, len(names))
|
||||||
|
for _, n := range names {
|
||||||
|
args = append(args, n)
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := db.QueryContext(ctx, `
|
||||||
|
SELECT id, fingerprint, labels, status, received_at
|
||||||
|
FROM alerts
|
||||||
|
WHERE name IN (`+placeholders(len(names))+`)
|
||||||
|
AND archived_at IS NULL`, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var out []deadmanAlert
|
||||||
|
for rows.Next() {
|
||||||
|
var a deadmanAlert
|
||||||
|
var labelsJSON, status string
|
||||||
|
if err := rows.Scan(&a.id, &a.fingerprint, &labelsJSON, &status, &a.receivedAt); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
json.Unmarshal([]byte(labelsJSON), &a.labels) //nolint:errcheck
|
||||||
|
|
||||||
|
m, ok := cfg.match(a.labels)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
a.matcher = m
|
||||||
|
a.resolved = status == "resolved"
|
||||||
|
out = append(out, a)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// deadmanDied raises the incident for a switch that has gone quiet.
|
||||||
|
//
|
||||||
|
// Two conditions gate it, and both matter. There must be no open incident for
|
||||||
|
// the switch already — the partial unique index enforces that anyway, but a
|
||||||
|
// second one would be a wasted page. And the heartbeat must have been seen since
|
||||||
|
// the last incident was raised, which is the re-arm rule: resolving a dead man's
|
||||||
|
// switch incident sticks, exactly as resolving an alert-backed one does (see
|
||||||
|
// incidentForGroup), and a source that is gone for good is a one-time page
|
||||||
|
// rather than a nag. Only a heartbeat that comes back and dies again earns a new
|
||||||
|
// incident.
|
||||||
|
func deadmanDied(ctx context.Context, db *sql.DB, cfg DeadmanConfig, notify NotifyConfig, sw deadmanAlert, now time.Time) error {
|
||||||
|
var lastTriggered, open int64
|
||||||
|
if err := db.QueryRowContext(ctx, `
|
||||||
|
SELECT COALESCE(MAX(triggered_at), 0),
|
||||||
|
COALESCE(SUM(resolved_at IS NULL), 0)
|
||||||
|
FROM incidents WHERE group_key = ?`,
|
||||||
|
sw.groupKey()).Scan(&lastTriggered, &open); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if open > 0 || sw.receivedAt <= lastTriggered {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
tx, err := db.BeginTx(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer tx.Rollback() //nolint:errcheck
|
||||||
|
|
||||||
|
// A heartbeat nobody has heard from is not firing, and saying otherwise in
|
||||||
|
// the alert list would be a lie. An Alertmanager-sourced resolution keeps its
|
||||||
|
// own source: it told us the truth first.
|
||||||
|
if !sw.resolved {
|
||||||
|
if _, err := tx.ExecContext(ctx, `
|
||||||
|
UPDATE alerts
|
||||||
|
SET status = 'resolved',
|
||||||
|
resolution_source = ?,
|
||||||
|
ends_at = COALESCE(ends_at, unixepoch())
|
||||||
|
WHERE id = ? AND status = 'firing'`, resolutionDeadman, sw.id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
severity := cfg.Severity
|
||||||
|
var sev *string
|
||||||
|
if severity != "" {
|
||||||
|
sev = &severity
|
||||||
|
}
|
||||||
|
|
||||||
|
incidentID, err := openIncident(ctx, tx, notify, sw.groupKey(),
|
||||||
|
"No heartbeat from "+sw.matcher.String(), sw.labels, sev)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
alertID := sw.id
|
||||||
|
detail := "last heartbeat " + humanDuration(now.Sub(time.Unix(sw.receivedAt, 0))) + " ago"
|
||||||
|
if err := logEvent(ctx, tx, incidentID, evDeadmanSilent, nil, &alertID, &detail); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Printf("deadman: %s went silent, opened incident %d", sw.matcher.String(), incidentID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// deadmanRecovered closes the incident for a switch that started chirping again.
|
||||||
|
//
|
||||||
|
// It cannot go through resolveIfSettled: a dead man's switch incident has no
|
||||||
|
// member alerts (linking the heartbeat would have the settled-incident cascade
|
||||||
|
// close it on the very same sweep that opened it), so the alert-driven cascade
|
||||||
|
// ignores it entirely and recovery is the only automatic way out.
|
||||||
|
func deadmanRecovered(ctx context.Context, db *sql.DB, sw deadmanAlert) error {
|
||||||
|
var incidentID int64
|
||||||
|
switch err := db.QueryRowContext(ctx, `
|
||||||
|
SELECT id FROM incidents
|
||||||
|
WHERE group_key = ? AND resolved_at IS NULL`, sw.groupKey()).Scan(&incidentID); {
|
||||||
|
case err == sql.ErrNoRows:
|
||||||
|
return nil
|
||||||
|
case err != nil:
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
tx, err := db.BeginTx(ctx, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer tx.Rollback() //nolint:errcheck
|
||||||
|
|
||||||
|
if _, err := tx.ExecContext(ctx, `
|
||||||
|
UPDATE incidents
|
||||||
|
SET status = 'resolved', resolved_at = ?, resolution_source = ?
|
||||||
|
WHERE id = ? AND resolved_at IS NULL`,
|
||||||
|
time.Now().Unix(), incidentResolutionRecovered, incidentID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := logEvent(ctx, tx, incidentID, evResolved, nil, nil, nil); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// The all-clear goes to whoever was paged, which enqueueResolved works out
|
||||||
|
// from the incident's own notification history.
|
||||||
|
if err := enqueueResolved(ctx, tx, incidentID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Printf("deadman: %s is back, resolved incident %d", sw.matcher.String(), incidentID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,468 @@
|
|||||||
|
package api_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Harness
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// watchdogGroupKey is what Alertmanager sends for a Watchdog grouped by
|
||||||
|
// alertname, which is how the deployed route is configured.
|
||||||
|
const watchdogGroupKey = `{}:{alertname="Watchdog"}`
|
||||||
|
|
||||||
|
// deadmanCfg watches Watchdog with a timeout short enough to reason about and
|
||||||
|
// long enough that a fresh heartbeat is never accidentally stale.
|
||||||
|
func deadmanCfg() api.DeadmanConfig {
|
||||||
|
return api.ParseDeadmanConfig("alertname=Watchdog", time.Hour, "critical")
|
||||||
|
}
|
||||||
|
|
||||||
|
// deadmanTS is notifyTS with dead man's switch handling on: notifications
|
||||||
|
// enabled against a fake ntfy, the admin on call today with a topic.
|
||||||
|
func deadmanTS(t *testing.T, cfg api.DeadmanConfig) (*ts, *fakeNtfy) {
|
||||||
|
t.Helper()
|
||||||
|
f := newFakeNtfy(t)
|
||||||
|
s := newDeadmanTS(t, cfg, api.NotifyConfig{
|
||||||
|
BaseURL: f.URL,
|
||||||
|
PublicURL: "https://terdut.example.com",
|
||||||
|
})
|
||||||
|
|
||||||
|
putOnCall(t, s, 1)
|
||||||
|
setTopic(t, s, 1, "terdut-admin")
|
||||||
|
return s, f
|
||||||
|
}
|
||||||
|
|
||||||
|
// heartbeat posts one Watchdog webhook. Its startsAt never changes: a dead man's
|
||||||
|
// switch alert fires once and is re-sent unchanged forever, which is precisely
|
||||||
|
// what makes its absence meaningful.
|
||||||
|
func heartbeat(t *testing.T, s *ts, fingerprint string, labels map[string]string) {
|
||||||
|
t.Helper()
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert(fingerprint, "Watchdog", "firing", "2026-05-20T10:00:00Z", zeroTime, labels),
|
||||||
|
}, watchdogGroupKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// silence back-dates a heartbeat's received_at, which is the only clock the
|
||||||
|
// sweeper reads. There is no fake clock in this package.
|
||||||
|
func silence(t *testing.T, s *ts, fingerprint string, ago time.Duration) {
|
||||||
|
t.Helper()
|
||||||
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = ?",
|
||||||
|
time.Now().Add(-ago).Unix(), fingerprint)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ageIncidents back-dates every incident. The re-arm rule compares a heartbeat
|
||||||
|
// against the last incident raised for its switch, so a test that wants a second
|
||||||
|
// episode has to put the first one in the past — there is no fake clock here.
|
||||||
|
func ageIncidents(t *testing.T, s *ts, ago time.Duration) {
|
||||||
|
t.Helper()
|
||||||
|
past := time.Now().Add(-ago).Unix()
|
||||||
|
s.exec(t, `UPDATE incidents
|
||||||
|
SET triggered_at = ?,
|
||||||
|
resolved_at = CASE WHEN resolved_at IS NULL THEN NULL ELSE ? END`,
|
||||||
|
past, past)
|
||||||
|
}
|
||||||
|
|
||||||
|
// incidentByGroup reads the incident for a group key, resolved ones included.
|
||||||
|
func incidentByGroup(t *testing.T, s *ts, groupKey string) (id int64, status, severity string, source *string) {
|
||||||
|
t.Helper()
|
||||||
|
err := s.db.QueryRow(`
|
||||||
|
SELECT id, status, COALESCE(severity, ''), resolution_source
|
||||||
|
FROM incidents WHERE group_key = ? ORDER BY id DESC LIMIT 1`,
|
||||||
|
groupKey).Scan(&id, &status, &severity, &source)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read incident for group %s: %v", groupKey, err)
|
||||||
|
}
|
||||||
|
return id, status, severity, source
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Receiving a heartbeat
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// The whole inversion: arrival of a dead man's switch alert is good news, and
|
||||||
|
// good news is not an incident.
|
||||||
|
func TestDeadman_HeartbeatOpensNoIncident(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, deadmanCfg())
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-watchdog", nil)
|
||||||
|
|
||||||
|
if got := s.countIncidents(t); got != 0 {
|
||||||
|
t.Fatalf("expected a heartbeat to open no incident, got %d", got)
|
||||||
|
}
|
||||||
|
if got := s.countNotifications(t, ""); got != 0 {
|
||||||
|
t.Errorf("expected no notification for a heartbeat, got %d", got)
|
||||||
|
}
|
||||||
|
if status, _, _ := s.alertRow(t, "fp-watchdog"); status != "firing" {
|
||||||
|
t.Errorf("expected the heartbeat to be stored firing, got %q", status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A heartbeat routed into a group alongside real alerts must not join their
|
||||||
|
// incident: it is not a symptom of anything.
|
||||||
|
func TestDeadman_MixedGroupExcludesHeartbeat(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, deadmanCfg())
|
||||||
|
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-mixed-wd", "Watchdog", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
amAlert("fp-mixed-disk", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
||||||
|
map[string]string{"severity": "critical"}),
|
||||||
|
}, `{}:{namespace="prod"}`)
|
||||||
|
|
||||||
|
if got := s.countIncidents(t); got != 1 {
|
||||||
|
t.Fatalf("expected 1 incident for the real alert, got %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
var alerts []map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/incidents/1/alerts", nil), &alerts)
|
||||||
|
if len(alerts) != 1 {
|
||||||
|
t.Fatalf("expected 1 member alert, got %d", len(alerts))
|
||||||
|
}
|
||||||
|
if name := alerts[0]["name"]; name != "DiskFull" {
|
||||||
|
t.Errorf("expected only the real alert linked, got %v", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A matcher scoped by label only claims the alerts it names, so a heartbeat from
|
||||||
|
// somewhere else stays an ordinary alert.
|
||||||
|
func TestDeadman_LabelScopedMatcherIgnoresOthers(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, api.ParseDeadmanConfig("alertname=Watchdog,cluster=prod", time.Hour, "critical"))
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-dev", map[string]string{"cluster": "dev"})
|
||||||
|
|
||||||
|
if got := s.countIncidents(t); got != 1 {
|
||||||
|
t.Fatalf("expected an unmatched Watchdog to behave like any other alert, got %d incidents", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Silence
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestDeadman_SilenceOpensIncident(t *testing.T) {
|
||||||
|
s, f := deadmanTS(t, deadmanCfg())
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-watchdog", nil)
|
||||||
|
silence(t, s, "fp-watchdog", 2*time.Hour)
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
|
||||||
|
if got := s.countIncidents(t); got != 1 {
|
||||||
|
t.Fatalf("expected silence to open 1 incident, got %d", got)
|
||||||
|
}
|
||||||
|
id, status, severity, _ := incidentByGroup(t, s, "deadman:fp-watchdog")
|
||||||
|
if status != "triggered" {
|
||||||
|
t.Errorf("expected a triggered incident, got %q", status)
|
||||||
|
}
|
||||||
|
if severity != "critical" {
|
||||||
|
t.Errorf("expected the configured severity, got %q", severity)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The alert list must not keep claiming a dead heartbeat is firing.
|
||||||
|
alertStatus, source, _ := s.alertRow(t, "fp-watchdog")
|
||||||
|
if alertStatus != "resolved" || source == nil || *source != "deadman" {
|
||||||
|
t.Errorf("expected the heartbeat resolved as deadman, got %q / %v", alertStatus, source)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nobody was told anything by an alert here, so the page has to come from
|
||||||
|
// the switch itself.
|
||||||
|
s.sweepNotify(t)
|
||||||
|
msgs := f.messages()
|
||||||
|
if len(msgs) != 1 {
|
||||||
|
t.Fatalf("expected 1 page, got %d", len(msgs))
|
||||||
|
}
|
||||||
|
if msgs[0].Topic != "terdut-admin" {
|
||||||
|
t.Errorf("expected the on-call topic, got %q", msgs[0].Topic)
|
||||||
|
}
|
||||||
|
if msgs[0].Priority != 5 {
|
||||||
|
t.Errorf("expected a critical page to override quiet hours (priority 5), got %d", msgs[0].Priority)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The timeline says why, with the age of the last heartbeat.
|
||||||
|
types := eventTypes(timeline(t, s, int(id)))
|
||||||
|
found := false
|
||||||
|
for _, ty := range types {
|
||||||
|
if ty == "deadman_silent" {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Errorf("expected a deadman_silent event, got %v", types)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The generic staleness sweep must keep its hands off heartbeats: they answer to
|
||||||
|
// their own, much tighter, timeout, and an 'expiry' resolution here would be
|
||||||
|
// both wrong and unrecoverable.
|
||||||
|
func TestDeadman_GenericExpiryLeavesHeartbeatAlone(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, deadmanCfg())
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-watchdog", nil)
|
||||||
|
silence(t, s, "fp-watchdog", 5*time.Minute)
|
||||||
|
|
||||||
|
// staleAfter far tighter than the dead man's switch timeout.
|
||||||
|
sweep(t, s, time.Minute)
|
||||||
|
|
||||||
|
status, source, _ := s.alertRow(t, "fp-watchdog")
|
||||||
|
if status != "firing" || source != nil {
|
||||||
|
t.Errorf("expected a live heartbeat left alone, got %q / %v", status, source)
|
||||||
|
}
|
||||||
|
if got := s.countIncidents(t); got != 0 {
|
||||||
|
t.Errorf("expected no incident for a heartbeat that is still fresh, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An explicit resolved from Alertmanager is the sender telling us the heartbeat
|
||||||
|
// stopped. There is nothing left to wait out.
|
||||||
|
func TestDeadman_AlertmanagerResolvedIsImmediateDeath(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, deadmanCfg())
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-watchdog", nil)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-watchdog", "Watchdog", "resolved", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
}, watchdogGroupKey)
|
||||||
|
|
||||||
|
// No ageing: received_at is seconds old, well inside the timeout.
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
|
||||||
|
if got := s.countIncidents(t); got != 1 {
|
||||||
|
t.Fatalf("expected a resolved heartbeat to open an incident at once, got %d", got)
|
||||||
|
}
|
||||||
|
// Alertmanager told the truth first, so its resolution source stands.
|
||||||
|
if _, source, _ := s.alertRow(t, "fp-watchdog"); source == nil || *source != "alertmanager" {
|
||||||
|
t.Errorf("expected the Alertmanager resolution source kept, got %v", source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Each label set is its own switch, so one healthy source cannot mask a dead one.
|
||||||
|
func TestDeadman_TracksEachFingerprintSeparately(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, deadmanCfg())
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-a", map[string]string{"cluster": "a"})
|
||||||
|
heartbeat(t, s, "fp-b", map[string]string{"cluster": "b"})
|
||||||
|
silence(t, s, "fp-b", 2*time.Hour)
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
|
||||||
|
if got := s.countIncidents(t); got != 1 {
|
||||||
|
t.Fatalf("expected only the silent switch to page, got %d incidents", got)
|
||||||
|
}
|
||||||
|
if _, status, _, _ := incidentByGroup(t, s, "deadman:fp-b"); status != "triggered" {
|
||||||
|
t.Errorf("expected the incident to belong to the silent switch, got %q", status)
|
||||||
|
}
|
||||||
|
if status, _, _ := s.alertRow(t, "fp-a"); status != "firing" {
|
||||||
|
t.Errorf("expected the live switch untouched, got %q", status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A switch nothing has ever been heard from is dormant. A fresh deploy, a
|
||||||
|
// restored database or a typo'd alertname must not page.
|
||||||
|
func TestDeadman_UnheardOfSwitchIsDormant(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, api.ParseDeadmanConfig("alertname=NeverSent", time.Hour, "critical"))
|
||||||
|
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
|
||||||
|
if got := s.countIncidents(t); got != 0 {
|
||||||
|
t.Fatalf("expected a switch that never chirped to be dormant, got %d incidents", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Recovery and re-arming
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// The returning heartbeat carries the unchanged startsAt of an alert that never
|
||||||
|
// stopped firing, so this also covers the ingest guard exemption: without it the
|
||||||
|
// upsert would discard the payload and the switch could die exactly once.
|
||||||
|
func TestDeadman_RecoveryResolvesIncident(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, deadmanCfg())
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-watchdog", nil)
|
||||||
|
silence(t, s, "fp-watchdog", 2*time.Hour)
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-watchdog", nil)
|
||||||
|
if status, source, _ := s.alertRow(t, "fp-watchdog"); status != "firing" || source != nil {
|
||||||
|
t.Fatalf("expected the returning heartbeat to be accepted, got %q / %v", status, source)
|
||||||
|
}
|
||||||
|
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
|
||||||
|
_, status, _, source := incidentByGroup(t, s, "deadman:fp-watchdog")
|
||||||
|
if status != "resolved" {
|
||||||
|
t.Errorf("expected recovery to close the incident, got %q", status)
|
||||||
|
}
|
||||||
|
if source == nil || *source != "recovered" {
|
||||||
|
t.Errorf("expected resolution_source recovered, got %v", source)
|
||||||
|
}
|
||||||
|
if got := s.countNotifications(t, "resolved"); got != 1 {
|
||||||
|
t.Errorf("expected 1 all-clear, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolving a dead man's switch incident sticks, exactly as it does for an
|
||||||
|
// alert-backed one. A source that is gone for good is a one-time page.
|
||||||
|
func TestDeadman_ManualResolveSticksWhileSilent(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, deadmanCfg())
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-watchdog", nil)
|
||||||
|
silence(t, s, "fp-watchdog", 2*time.Hour)
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
|
||||||
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
||||||
|
|
||||||
|
// Still silent, several sweeps later.
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
|
||||||
|
if got := s.countIncidents(t); got != 1 {
|
||||||
|
t.Fatalf("expected a manually resolved incident to stay closed, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ...but the switch re-arms, so a heartbeat that comes back and dies again is a
|
||||||
|
// new incident rather than silence forever.
|
||||||
|
func TestDeadman_ReArmsAfterHeartbeatReturns(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, deadmanCfg())
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-watchdog", nil)
|
||||||
|
silence(t, s, "fp-watchdog", 2*time.Hour)
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
||||||
|
|
||||||
|
// That episode is yesterday's news; the heartbeat now returns after it.
|
||||||
|
ageIncidents(t, s, 10*time.Hour)
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-watchdog", nil)
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
if got := s.countIncidents(t); got != 1 {
|
||||||
|
t.Fatalf("expected the live switch to open nothing, got %d incidents", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
silence(t, s, "fp-watchdog", 2*time.Hour)
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
|
||||||
|
if got := s.countIncidents(t); got != 2 {
|
||||||
|
t.Fatalf("expected a second death to open a second incident, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A dead man's switch incident has no member alerts — linking the heartbeat
|
||||||
|
// would have the settled-incident cascade close it on the very sweep that opened
|
||||||
|
// it — so the cascade has to leave it alone.
|
||||||
|
func TestDeadman_SettledCascadeLeavesIncidentOpen(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, deadmanCfg())
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-watchdog", nil)
|
||||||
|
silence(t, s, "fp-watchdog", 2*time.Hour)
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
sweep(t, s, noArchive)
|
||||||
|
|
||||||
|
if _, status, _, _ := incidentByGroup(t, s, "deadman:fp-watchdog"); status != "triggered" {
|
||||||
|
t.Fatalf("expected the incident to stay open until the switch recovers, got %q", status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Configuration
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestParseDeadmanConfig(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
matchers string
|
||||||
|
timeout time.Duration
|
||||||
|
want []api.DeadmanMatcher
|
||||||
|
enabled bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "single alertname",
|
||||||
|
matchers: "alertname=Watchdog",
|
||||||
|
timeout: time.Hour,
|
||||||
|
want: []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}},
|
||||||
|
enabled: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "several matchers with extra labels and whitespace",
|
||||||
|
matchers: " alertname=Watchdog, cluster=prod ; alertname=EdgeHeartbeat ",
|
||||||
|
timeout: time.Hour,
|
||||||
|
want: []api.DeadmanMatcher{
|
||||||
|
{Name: "Watchdog", Labels: map[string]string{"cluster": "prod"}},
|
||||||
|
{Name: "EdgeHeartbeat", Labels: map[string]string{}},
|
||||||
|
},
|
||||||
|
enabled: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// Mandatory: it is what keeps the sweeper's candidate query on an index.
|
||||||
|
name: "matcher without alertname is dropped",
|
||||||
|
matchers: "cluster=prod; alertname=Watchdog",
|
||||||
|
timeout: time.Hour,
|
||||||
|
want: []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}},
|
||||||
|
enabled: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "malformed condition drops only its matcher",
|
||||||
|
matchers: "alertname=Watchdog,garbage; alertname=Other",
|
||||||
|
timeout: time.Hour,
|
||||||
|
want: []api.DeadmanMatcher{{Name: "Other", Labels: map[string]string{}}},
|
||||||
|
enabled: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "zero timeout disables",
|
||||||
|
matchers: "alertname=Watchdog",
|
||||||
|
timeout: 0,
|
||||||
|
want: []api.DeadmanMatcher{{Name: "Watchdog", Labels: map[string]string{}}},
|
||||||
|
enabled: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no usable matchers disables",
|
||||||
|
matchers: "",
|
||||||
|
timeout: time.Hour,
|
||||||
|
want: nil,
|
||||||
|
enabled: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got := api.ParseDeadmanConfig(tc.matchers, tc.timeout, "critical")
|
||||||
|
if len(got.Matchers) != len(tc.want) {
|
||||||
|
t.Fatalf("got %d matchers %v, want %d", len(got.Matchers), got.Matchers, len(tc.want))
|
||||||
|
}
|
||||||
|
for i, w := range tc.want {
|
||||||
|
if got.Matchers[i].Name != w.Name {
|
||||||
|
t.Errorf("matcher %d: name %q, want %q", i, got.Matchers[i].Name, w.Name)
|
||||||
|
}
|
||||||
|
if len(got.Matchers[i].Labels) != len(w.Labels) {
|
||||||
|
t.Errorf("matcher %d: labels %v, want %v", i, got.Matchers[i].Labels, w.Labels)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for k, v := range w.Labels {
|
||||||
|
if got.Matchers[i].Labels[k] != v {
|
||||||
|
t.Errorf("matcher %d: label %s=%q, want %q", i, k, got.Matchers[i].Labels[k], v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A zero config is off, which is what keeps the feature opt-in for anything
|
||||||
|
// building a router without one.
|
||||||
|
func TestDeadman_DisabledConfigIsInert(t *testing.T) {
|
||||||
|
s, _ := deadmanTS(t, api.DeadmanConfig{})
|
||||||
|
|
||||||
|
heartbeat(t, s, "fp-watchdog", nil)
|
||||||
|
silence(t, s, "fp-watchdog", 48*time.Hour)
|
||||||
|
sweep(t, s, time.Hour)
|
||||||
|
|
||||||
|
// Ordinary alert handling: an incident from the arrival, not the absence.
|
||||||
|
if got := s.countIncidents(t); got != 1 {
|
||||||
|
t.Fatalf("expected plain alert handling with deadman off, got %d incidents", got)
|
||||||
|
}
|
||||||
|
if _, source, _ := s.alertRow(t, "fp-watchdog"); source == nil || *source != "expiry" {
|
||||||
|
t.Errorf("expected the generic sweeper to own the alert, got %v", source)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,300 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Values for incidents.resolution_source, recording who closed the incident:
|
||||||
|
// every member alert stopped firing, or a person decided it was done.
|
||||||
|
const (
|
||||||
|
incidentResolutionAlerts = "alerts"
|
||||||
|
incidentResolutionManual = "manual"
|
||||||
|
|
||||||
|
// incidentResolutionRecovered closes a dead man's switch incident whose
|
||||||
|
// heartbeat started arriving again. It cannot be "alerts": these incidents
|
||||||
|
// have no member alerts for the cascade to work from.
|
||||||
|
incidentResolutionRecovered = "recovered"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Incident timeline event types. Stored as free text so adding one later is not
|
||||||
|
// a migration, but these are the ones the server writes.
|
||||||
|
const (
|
||||||
|
evTriggered = "triggered"
|
||||||
|
evAlertAdded = "alert_added"
|
||||||
|
evAlertResolved = "alert_resolved"
|
||||||
|
evAcknowledged = "acknowledged"
|
||||||
|
evUnacknowledged = "unacknowledged"
|
||||||
|
evAssigned = "assigned"
|
||||||
|
evSnoozed = "snoozed"
|
||||||
|
evUnsnoozed = "unsnoozed"
|
||||||
|
evResolved = "resolved"
|
||||||
|
evNote = "note"
|
||||||
|
evDeadmanSilent = "deadman_silent"
|
||||||
|
)
|
||||||
|
|
||||||
|
// severityLabel is the Alertmanager label an incident's severity is derived from.
|
||||||
|
const severityLabel = "severity"
|
||||||
|
|
||||||
|
// querier is satisfied by both *sql.DB and *sql.Tx, so the helpers below work
|
||||||
|
// inside the webhook's transaction and standalone from handlers and the sweeper.
|
||||||
|
type querier interface {
|
||||||
|
ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
|
||||||
|
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
||||||
|
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
||||||
|
}
|
||||||
|
|
||||||
|
const incidentSelectFrom = `
|
||||||
|
SELECT i.id, i.group_key, i.title, i.group_labels, i.status, i.severity,
|
||||||
|
i.triggered_at,
|
||||||
|
i.acknowledged_by, i.acknowledged_at, ack.username,
|
||||||
|
i.assigned_to, asg.username, i.snoozed_until,
|
||||||
|
i.resolved_at, i.resolution_source, i.archived_at
|
||||||
|
FROM incidents i
|
||||||
|
LEFT JOIN users ack ON ack.id = i.acknowledged_by
|
||||||
|
LEFT JOIN users asg ON asg.id = i.assigned_to`
|
||||||
|
|
||||||
|
func scanIncident(s scanner) (models.Incident, error) {
|
||||||
|
var i models.Incident
|
||||||
|
var groupLabelsJSON string
|
||||||
|
var triggeredAt int64
|
||||||
|
var ackAt, snoozedUntil, resolvedAt, archivedAt *int64
|
||||||
|
|
||||||
|
if err := s.Scan(
|
||||||
|
&i.ID, &i.GroupKey, &i.Title, &groupLabelsJSON, &i.Status, &i.Severity,
|
||||||
|
&triggeredAt,
|
||||||
|
&i.AcknowledgedByID, &ackAt, &i.AcknowledgedByUser,
|
||||||
|
&i.AssignedToID, &i.AssignedToUser, &snoozedUntil,
|
||||||
|
&resolvedAt, &i.ResolutionSource, &archivedAt,
|
||||||
|
); err != nil {
|
||||||
|
return i, err
|
||||||
|
}
|
||||||
|
|
||||||
|
json.Unmarshal([]byte(groupLabelsJSON), &i.GroupLabels) //nolint:errcheck
|
||||||
|
i.TriggeredAt = time.Unix(triggeredAt, 0).UTC()
|
||||||
|
i.AcknowledgedAt = unixPtr(ackAt)
|
||||||
|
i.SnoozedUntil = unixPtr(snoozedUntil)
|
||||||
|
i.ResolvedAt = unixPtr(resolvedAt)
|
||||||
|
i.ArchivedAt = unixPtr(archivedAt)
|
||||||
|
return i, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// unixPtr converts a nullable Unix-second column to a nullable UTC time.
|
||||||
|
func unixPtr(sec *int64) *time.Time {
|
||||||
|
if sec == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
t := time.Unix(*sec, 0).UTC()
|
||||||
|
return &t
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchIncident(ctx context.Context, q querier, id int64) (models.Incident, error) {
|
||||||
|
return scanIncident(q.QueryRowContext(ctx, incidentSelectFrom+" WHERE i.id = ?", id))
|
||||||
|
}
|
||||||
|
|
||||||
|
// logEvent appends one entry to an incident's timeline. A nil userID means the
|
||||||
|
// server acted rather than a person.
|
||||||
|
func logEvent(ctx context.Context, q querier, incidentID int64, evType string, userID, alertID *int64, detail *string) error {
|
||||||
|
_, err := q.ExecContext(ctx, `
|
||||||
|
INSERT INTO incident_events (incident_id, type, user_id, alert_id, detail, created_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||||
|
incidentID, evType, userID, alertID, detail, time.Now().Unix())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// todayUTC is the schedule's day key. The schedule's smallest unit is one UTC day.
|
||||||
|
func todayUTC() string {
|
||||||
|
return time.Now().UTC().Format("2006-01-02")
|
||||||
|
}
|
||||||
|
|
||||||
|
// currentOnCall returns today's on-call user, or nil when nobody is scheduled.
|
||||||
|
// A missing schedule entry is not an error — incidents just open unassigned.
|
||||||
|
func currentOnCall(ctx context.Context, q querier) (*int64, error) {
|
||||||
|
var userID int64
|
||||||
|
err := q.QueryRowContext(ctx,
|
||||||
|
"SELECT user_id FROM schedule_entries WHERE date = ?", todayUTC()).Scan(&userID)
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &userID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// severityRank orders the conventional Alertmanager severity label values.
|
||||||
|
// Anything unrecognised sorts below all of them rather than being dropped.
|
||||||
|
func severityRank(s string) int {
|
||||||
|
switch strings.ToLower(s) {
|
||||||
|
case "critical":
|
||||||
|
return 4
|
||||||
|
case "error":
|
||||||
|
return 3
|
||||||
|
case "warning":
|
||||||
|
return 2
|
||||||
|
case "info":
|
||||||
|
return 1
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// refreshSeverity raises an incident's severity to the highest `severity` label
|
||||||
|
// seen across its alerts.
|
||||||
|
//
|
||||||
|
// It is a high-water mark, never lowered: an incident that hit critical was a
|
||||||
|
// critical incident, even after the critical alert clears and a warning is all
|
||||||
|
// that is left firing. Downgrading a live incident would also quietly demote it
|
||||||
|
// in the queue while the work is still open.
|
||||||
|
func refreshSeverity(ctx context.Context, q querier, incidentID int64) error {
|
||||||
|
rows, err := q.QueryContext(ctx, `
|
||||||
|
SELECT json_extract(a.labels, '$.'||?)
|
||||||
|
FROM incident_alerts ia
|
||||||
|
JOIN alerts a ON a.id = ia.alert_id
|
||||||
|
WHERE ia.incident_id = ?`, severityLabel, incidentID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
best := ""
|
||||||
|
for rows.Next() {
|
||||||
|
var sev *string
|
||||||
|
if err := rows.Scan(&sev); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if sev != nil && severityRank(*sev) > severityRank(best) {
|
||||||
|
best = *sev
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
|
||||||
|
if best == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// The comparison lives in SQL so an unrelated concurrent update cannot be
|
||||||
|
// clobbered by a stale read.
|
||||||
|
_, err = q.ExecContext(ctx, `
|
||||||
|
UPDATE incidents SET severity = ?
|
||||||
|
WHERE id = ?
|
||||||
|
AND (severity IS NULL OR `+severityRankSQL("severity")+` < ?)`,
|
||||||
|
best, incidentID, severityRank(best))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// severityRankSQL mirrors severityRank for use inside a statement. SQL cannot
|
||||||
|
// order these strings meaningfully on its own.
|
||||||
|
func severityRankSQL(col string) string {
|
||||||
|
return `CASE lower(COALESCE(` + col + `, ''))
|
||||||
|
WHEN 'critical' THEN 4
|
||||||
|
WHEN 'error' THEN 3
|
||||||
|
WHEN 'warning' THEN 2
|
||||||
|
WHEN 'info' THEN 1
|
||||||
|
ELSE 0 END`
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveIfSettled closes an incident once every alert under it has stopped
|
||||||
|
// firing — PagerDuty's cascade, and the only automatic route out of the open
|
||||||
|
// state. Reports whether it actually resolved anything.
|
||||||
|
func resolveIfSettled(ctx context.Context, q querier, incidentID int64) (bool, error) {
|
||||||
|
res, err := q.ExecContext(ctx, `
|
||||||
|
UPDATE incidents
|
||||||
|
SET status = 'resolved',
|
||||||
|
resolved_at = ?,
|
||||||
|
resolution_source = ?
|
||||||
|
WHERE id = ?
|
||||||
|
AND resolved_at IS NULL
|
||||||
|
-- An incident with no members yet is mid-creation, not settled.
|
||||||
|
AND EXISTS (SELECT 1 FROM incident_alerts ia WHERE ia.incident_id = incidents.id)
|
||||||
|
AND NOT EXISTS (SELECT 1
|
||||||
|
FROM incident_alerts ia
|
||||||
|
JOIN alerts a ON a.id = ia.alert_id
|
||||||
|
WHERE ia.incident_id = incidents.id
|
||||||
|
AND a.status = 'firing')`,
|
||||||
|
time.Now().Unix(), incidentResolutionAlerts, incidentID)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
n, _ := res.RowsAffected()
|
||||||
|
if n == 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err := logEvent(ctx, q, incidentID, evResolved, nil, nil, nil); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
// The all-clear goes only to whoever was paged in the first place, which
|
||||||
|
// enqueueResolved works out from the incident's own notification history.
|
||||||
|
// Manual resolution sends nothing: the person who closed it already knows.
|
||||||
|
return true, enqueueResolved(ctx, q, incidentID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// acknowledgeIncident records that userID has picked an incident up, and reports
|
||||||
|
// whether it changed anything — an already-resolved incident is left alone.
|
||||||
|
// Shared by the authenticated handler and the Acknowledge button in a push
|
||||||
|
// notification, so both write the same state and the same timeline entry.
|
||||||
|
func acknowledgeIncident(ctx context.Context, q querier, incidentID, userID int64) (bool, error) {
|
||||||
|
res, err := q.ExecContext(ctx, `
|
||||||
|
UPDATE incidents
|
||||||
|
SET status = 'acknowledged', acknowledged_by = ?, acknowledged_at = ?
|
||||||
|
WHERE id = ? AND resolved_at IS NULL`,
|
||||||
|
userID, time.Now().Unix(), incidentID)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n == 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return true, logEvent(ctx, q, incidentID, evAcknowledged, &userID, nil, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// openIncidentForAlert returns the open incident an alert currently belongs to,
|
||||||
|
// or 0 when it has none. Used when an alert resolves or expires so the event
|
||||||
|
// lands on the right timeline.
|
||||||
|
func openIncidentForAlert(ctx context.Context, q querier, alertID int64) (int64, error) {
|
||||||
|
var id int64
|
||||||
|
err := q.QueryRowContext(ctx, `
|
||||||
|
SELECT i.id
|
||||||
|
FROM incident_alerts ia
|
||||||
|
JOIN incidents i ON i.id = ia.incident_id
|
||||||
|
WHERE ia.alert_id = ? AND i.resolved_at IS NULL`, alertID).Scan(&id)
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
return id, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// incidentTitle renders a human-readable title from Alertmanager's groupLabels,
|
||||||
|
// leading with the alert name and appending whatever else the operator grouped
|
||||||
|
// by. Falls back to the alert's own name when the payload carried no groupLabels.
|
||||||
|
func incidentTitle(groupLabels map[string]string, fallback string) string {
|
||||||
|
name := groupLabels["alertname"]
|
||||||
|
if name == "" {
|
||||||
|
name = fallback
|
||||||
|
}
|
||||||
|
if name == "" {
|
||||||
|
name = "Incident"
|
||||||
|
}
|
||||||
|
|
||||||
|
rest := make([]string, 0, len(groupLabels))
|
||||||
|
for k, v := range groupLabels {
|
||||||
|
if k == "alertname" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rest = append(rest, k+"="+v)
|
||||||
|
}
|
||||||
|
if len(rest) == 0 {
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
sort.Strings(rest)
|
||||||
|
return name + " (" + strings.Join(rest, ", ") + ")"
|
||||||
|
}
|
||||||
@@ -0,0 +1,557 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/models"
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
func handleListIncidents(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
q := r.URL.Query()
|
||||||
|
|
||||||
|
where := []string{}
|
||||||
|
args := []any{}
|
||||||
|
|
||||||
|
// Without an explicit status the queue shows open work, which is what an
|
||||||
|
// on-call person opens the tool to see.
|
||||||
|
if status := q.Get("status"); status != "" {
|
||||||
|
where = append(where, "i.status = ?")
|
||||||
|
args = append(args, status)
|
||||||
|
} else {
|
||||||
|
where = append(where, "i.resolved_at IS NULL")
|
||||||
|
}
|
||||||
|
|
||||||
|
if q.Get("archived") == "true" {
|
||||||
|
where = append(where, "i.archived_at IS NOT NULL")
|
||||||
|
} else {
|
||||||
|
where = append(where, "i.archived_at IS NULL")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A snooze expires by simply falling into the past; nothing sweeps it.
|
||||||
|
if q.Get("snoozed") == "true" {
|
||||||
|
where = append(where, "i.snoozed_until > ?")
|
||||||
|
args = append(args, time.Now().Unix())
|
||||||
|
} else {
|
||||||
|
where = append(where, "(i.snoozed_until IS NULL OR i.snoozed_until <= ?)")
|
||||||
|
args = append(args, time.Now().Unix())
|
||||||
|
}
|
||||||
|
|
||||||
|
if severity := q.Get("severity"); severity != "" {
|
||||||
|
where = append(where, "i.severity = ?")
|
||||||
|
args = append(args, severity)
|
||||||
|
}
|
||||||
|
if assignee := q.Get("assigned_to"); assignee != "" {
|
||||||
|
if n, err := strconv.ParseInt(assignee, 10, 64); err == nil {
|
||||||
|
where = append(where, "i.assigned_to = ?")
|
||||||
|
args = append(args, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if from := q.Get("from"); from != "" {
|
||||||
|
if t, err := time.Parse("2006-01-02", from); err == nil {
|
||||||
|
where = append(where, "i.triggered_at >= ?")
|
||||||
|
args = append(args, t.UTC().Unix())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if to := q.Get("to"); to != "" {
|
||||||
|
if t, err := time.Parse("2006-01-02", to); err == nil {
|
||||||
|
where = append(where, "i.triggered_at < ?")
|
||||||
|
args = append(args, t.UTC().AddDate(0, 0, 1).Unix())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
limit := 50
|
||||||
|
if l := q.Get("limit"); l != "" {
|
||||||
|
if n, err := strconv.Atoi(l); err == nil && n > 0 && n <= 500 {
|
||||||
|
limit = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
order := "i.triggered_at DESC"
|
||||||
|
if q.Get("sort") == "severity" {
|
||||||
|
order = severityRankSQL("i.severity") + " DESC, i.triggered_at DESC"
|
||||||
|
}
|
||||||
|
args = append(args, limit)
|
||||||
|
|
||||||
|
rows, err := db.QueryContext(r.Context(),
|
||||||
|
fmt.Sprintf("%s WHERE %s ORDER BY %s LIMIT ?",
|
||||||
|
incidentSelectFrom, strings.Join(where, " AND "), order),
|
||||||
|
args...)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
incidents := []models.Incident{}
|
||||||
|
for rows.Next() {
|
||||||
|
i, err := scanIncident(rows)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
incidents = append(incidents, i)
|
||||||
|
}
|
||||||
|
respond(w, http.StatusOK, incidents)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleGetIncident(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
inc, err := fetchIncident(r.Context(), db, id)
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
respond(w, http.StatusNotFound, errResp("incident not found"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if inc.Alerts, err = incidentAlerts(r, db, id); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respond(w, http.StatusOK, inc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleIncidentAlerts(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !incidentExists(w, r, db, id) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
alerts, err := incidentAlerts(r, db, id)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respond(w, http.StatusOK, alerts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleIncidentTimeline(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !incidentExists(w, r, db, id) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := db.QueryContext(r.Context(), `
|
||||||
|
SELECT e.id, e.incident_id, e.type, e.user_id, u.username,
|
||||||
|
e.alert_id, e.detail, e.created_at
|
||||||
|
FROM incident_events e
|
||||||
|
LEFT JOIN users u ON u.id = e.user_id
|
||||||
|
WHERE e.incident_id = ?
|
||||||
|
ORDER BY e.created_at ASC, e.id ASC`, id)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
events := []models.IncidentEvent{}
|
||||||
|
for rows.Next() {
|
||||||
|
var e models.IncidentEvent
|
||||||
|
var ts int64
|
||||||
|
if err := rows.Scan(&e.ID, &e.IncidentID, &e.Type, &e.UserID, &e.Username,
|
||||||
|
&e.AlertID, &e.Detail, &ts); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
e.CreatedAt = time.Unix(ts, 0).UTC()
|
||||||
|
events = append(events, e)
|
||||||
|
}
|
||||||
|
respond(w, http.StatusOK, events)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleIncidentAcknowledge(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
user, _ := userFromContext(r.Context())
|
||||||
|
acked, err := acknowledgeIncident(r.Context(), db, id, user.ID)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !acked {
|
||||||
|
if !incidentExists(w, r, db, id) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respond(w, http.StatusConflict, errResp("incident is resolved"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respondIncident(w, r, db, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleIncidentUnacknowledge(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
user, _ := userFromContext(r.Context())
|
||||||
|
if !updateOpenIncident(w, r, db, id,
|
||||||
|
`UPDATE incidents SET status = 'triggered', acknowledged_by = NULL, acknowledged_at = NULL
|
||||||
|
WHERE id = ? AND resolved_at IS NULL`, id) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := logEvent(r.Context(), db, id, evUnacknowledged, &user.ID, nil, nil); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleIncidentResolve closes an incident by hand. This is terminal: a later
|
||||||
|
// occurrence opens a new incident rather than reopening this one, which is what
|
||||||
|
// stops a resolved incident from reappearing on the next repeat_interval
|
||||||
|
// re-send of an alert that never stopped firing. Use snooze for "not now".
|
||||||
|
func handleIncidentResolve(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
user, _ := userFromContext(r.Context())
|
||||||
|
if !updateOpenIncident(w, r, db, id,
|
||||||
|
`UPDATE incidents SET status = 'resolved', resolved_at = ?, resolution_source = ?
|
||||||
|
WHERE id = ? AND resolved_at IS NULL`,
|
||||||
|
time.Now().Unix(), incidentResolutionManual, id) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := logEvent(r.Context(), db, id, evResolved, &user.ID, nil, nil); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respondIncident(w, r, db, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleIncidentAssign(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req struct {
|
||||||
|
UserID int64 `json:"user_id"`
|
||||||
|
}
|
||||||
|
if err := decodeJSON(r, &req); err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if req.UserID == 0 {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("user_id is required"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var exists int
|
||||||
|
if err := db.QueryRowContext(r.Context(),
|
||||||
|
"SELECT 1 FROM users WHERE id = ?", req.UserID).Scan(&exists); err != nil {
|
||||||
|
respond(w, http.StatusNotFound, errResp("user not found"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if !updateOpenIncident(w, r, db, id,
|
||||||
|
"UPDATE incidents SET assigned_to = ? WHERE id = ? AND resolved_at IS NULL",
|
||||||
|
req.UserID, id) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// On an "assigned" event user_id is the assignee, not the actor.
|
||||||
|
if err := logEvent(r.Context(), db, id, evAssigned, &req.UserID, nil, nil); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respondIncident(w, r, db, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleIncidentSnooze hides an incident from the default queue without closing
|
||||||
|
// it. Accepts either an absolute {"until": RFC3339} or a relative
|
||||||
|
// {"duration": "2h"}.
|
||||||
|
func handleIncidentSnooze(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req struct {
|
||||||
|
Until string `json:"until"`
|
||||||
|
Duration string `json:"duration"`
|
||||||
|
}
|
||||||
|
if err := decodeJSON(r, &req); err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var until time.Time
|
||||||
|
switch {
|
||||||
|
case req.Until != "":
|
||||||
|
t, err := time.Parse(time.RFC3339, req.Until)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid until (expected RFC3339)"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
until = t
|
||||||
|
case req.Duration != "":
|
||||||
|
d, err := time.ParseDuration(req.Duration)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid duration"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
until = time.Now().Add(d)
|
||||||
|
default:
|
||||||
|
respond(w, http.StatusBadRequest, errResp("until or duration is required"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !until.After(time.Now()) {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("snooze must end in the future"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
user, _ := userFromContext(r.Context())
|
||||||
|
if !updateOpenIncident(w, r, db, id,
|
||||||
|
"UPDATE incidents SET snoozed_until = ? WHERE id = ? AND resolved_at IS NULL",
|
||||||
|
until.Unix(), id) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
detail := until.UTC().Format(time.RFC3339)
|
||||||
|
if err := logEvent(r.Context(), db, id, evSnoozed, &user.ID, nil, &detail); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respondIncident(w, r, db, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleIncidentUnsnooze(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
user, _ := userFromContext(r.Context())
|
||||||
|
if !updateOpenIncident(w, r, db, id,
|
||||||
|
"UPDATE incidents SET snoozed_until = NULL WHERE id = ? AND resolved_at IS NULL", id) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := logEvent(r.Context(), db, id, evUnsnoozed, &user.ID, nil, nil); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleIncidentArchive(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
res, err := db.ExecContext(r.Context(),
|
||||||
|
"UPDATE incidents SET archived_at = unixepoch() WHERE id = ?", id)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n == 0 {
|
||||||
|
respond(w, http.StatusNotFound, errResp("incident not found"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respondIncident(w, r, db, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleIncidentUnarchive(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
res, err := db.ExecContext(r.Context(),
|
||||||
|
"UPDATE incidents SET archived_at = NULL WHERE id = ?", id)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n == 0 {
|
||||||
|
respond(w, http.StatusNotFound, errResp("incident not found"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleCreateNote adds a note to the timeline. Notes are ordinary events, so a
|
||||||
|
// single query renders the whole story of an incident in order.
|
||||||
|
func handleCreateNote(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
}
|
||||||
|
if err := decodeJSON(r, &req); err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if req.Content == "" {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("content is required"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !incidentExists(w, r, db, id) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
user, _ := userFromContext(r.Context())
|
||||||
|
now := time.Now()
|
||||||
|
res, err := db.ExecContext(r.Context(), `
|
||||||
|
INSERT INTO incident_events (incident_id, type, user_id, detail, created_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?)`, id, evNote, user.ID, req.Content, now.Unix())
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
eventID, _ := res.LastInsertId()
|
||||||
|
|
||||||
|
respond(w, http.StatusCreated, models.IncidentEvent{
|
||||||
|
ID: eventID,
|
||||||
|
IncidentID: id,
|
||||||
|
Type: evNote,
|
||||||
|
UserID: &user.ID,
|
||||||
|
Username: &user.Username,
|
||||||
|
Detail: &req.Content,
|
||||||
|
CreatedAt: now.UTC().Truncate(time.Second),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleDeleteNote removes one of your own notes. Only notes are deletable — the
|
||||||
|
// rest of the timeline is what actually happened, and is not editable.
|
||||||
|
func handleDeleteNote(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, ok := incidentIDParam(w, r)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
eventID, err := strconv.ParseInt(chi.URLParam(r, "eventID"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid note id"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
user, _ := userFromContext(r.Context())
|
||||||
|
res, err := db.ExecContext(r.Context(), `
|
||||||
|
DELETE FROM incident_events
|
||||||
|
WHERE id = ? AND incident_id = ? AND type = ? AND user_id = ?`,
|
||||||
|
eventID, id, evNote, user.ID)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n == 0 {
|
||||||
|
respond(w, http.StatusNotFound, errResp("note not found"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Shared handler plumbing
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func incidentIDParam(w http.ResponseWriter, r *http.Request) (int64, bool) {
|
||||||
|
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid incident id"))
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return id, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func incidentExists(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64) bool {
|
||||||
|
var exists int
|
||||||
|
if err := db.QueryRowContext(r.Context(),
|
||||||
|
"SELECT 1 FROM incidents WHERE id = ?", id).Scan(&exists); err != nil {
|
||||||
|
respond(w, http.StatusNotFound, errResp("incident not found"))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// updateOpenIncident runs a mutation that is only valid while an incident is
|
||||||
|
// open. The query must be constrained to `resolved_at IS NULL`, so no rows means
|
||||||
|
// either the incident does not exist or it is already closed — two different
|
||||||
|
// answers the caller should not have to distinguish itself.
|
||||||
|
func updateOpenIncident(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64, query string, args ...any) bool {
|
||||||
|
res, err := db.ExecContext(r.Context(), query, args...)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if !incidentExists(w, r, db, id) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
respond(w, http.StatusConflict, errResp("incident is resolved"))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func respondIncident(w http.ResponseWriter, r *http.Request, db *sql.DB, id int64) {
|
||||||
|
inc, err := fetchIncident(r.Context(), db, id)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respond(w, http.StatusOK, inc)
|
||||||
|
}
|
||||||
|
|
||||||
|
// incidentAlerts loads the alerts under an incident, newest signal first.
|
||||||
|
func incidentAlerts(r *http.Request, db *sql.DB, id int64) ([]models.Alert, error) {
|
||||||
|
rows, err := db.QueryContext(r.Context(), alertSelectFrom+`
|
||||||
|
JOIN incident_alerts m ON m.alert_id = a.id
|
||||||
|
WHERE m.incident_id = ?
|
||||||
|
ORDER BY a.received_at DESC`, id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
alerts := []models.Alert{}
|
||||||
|
for rows.Next() {
|
||||||
|
a, err := scanAlert(rows)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
alerts = append(alerts, a)
|
||||||
|
}
|
||||||
|
return alerts, rows.Err()
|
||||||
|
}
|
||||||
@@ -0,0 +1,827 @@
|
|||||||
|
package api_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/api"
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
// amAlert builds one alert of a webhook payload.
|
||||||
|
func amAlert(fingerprint, name, status, startsAt, endsAt string, labels map[string]string) map[string]any {
|
||||||
|
l := map[string]string{"alertname": name}
|
||||||
|
for k, v := range labels {
|
||||||
|
l[k] = v
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"status": status,
|
||||||
|
"labels": l,
|
||||||
|
"annotations": map[string]string{},
|
||||||
|
"startsAt": startsAt,
|
||||||
|
"endsAt": endsAt,
|
||||||
|
"generatorURL": "",
|
||||||
|
"fingerprint": fingerprint,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func listIncidents(t *testing.T, s *ts, query string) []map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
var out []map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/incidents"+query, nil), &out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func getIncident(t *testing.T, s *ts, id int) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
var out map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d", id), nil), &out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func timeline(t *testing.T, s *ts, id int) []map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
var out []map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, fmt.Sprintf("/api/incidents/%d/timeline", id), nil), &out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventTypes flattens a timeline to its event types, which is what the ordering
|
||||||
|
// assertions actually care about.
|
||||||
|
func eventTypes(events []map[string]any) []string {
|
||||||
|
types := make([]string, len(events))
|
||||||
|
for i, e := range events {
|
||||||
|
types[i] = e["type"].(string)
|
||||||
|
}
|
||||||
|
return types
|
||||||
|
}
|
||||||
|
|
||||||
|
// countIncidents counts rows directly, including resolved and archived ones that
|
||||||
|
// no list view returns.
|
||||||
|
func (s *ts) countIncidents(t *testing.T) int {
|
||||||
|
t.Helper()
|
||||||
|
var n int
|
||||||
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM incidents").Scan(&n); err != nil {
|
||||||
|
t.Fatalf("count incidents: %v", err)
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Ingest: alerts becoming incidents
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestWebhook_FiringOpensIncident(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-1", "HighCPU", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
||||||
|
map[string]string{"severity": "critical"}),
|
||||||
|
}, "{}:{alertname=\"HighCPU\"}")
|
||||||
|
|
||||||
|
incidents := listIncidents(t, s, "")
|
||||||
|
if len(incidents) != 1 {
|
||||||
|
t.Fatalf("expected 1 incident, got %d", len(incidents))
|
||||||
|
}
|
||||||
|
inc := incidents[0]
|
||||||
|
if inc["status"] != "triggered" {
|
||||||
|
t.Errorf("expected status triggered, got %v", inc["status"])
|
||||||
|
}
|
||||||
|
if inc["severity"] != "critical" {
|
||||||
|
t.Errorf("expected severity critical, got %v", inc["severity"])
|
||||||
|
}
|
||||||
|
if inc["title"] != "HighCPU" {
|
||||||
|
t.Errorf("expected title from groupLabels, got %v", inc["title"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// The alert points back at the incident it opened.
|
||||||
|
var alerts []map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/alerts", nil), &alerts)
|
||||||
|
if len(alerts) != 1 || alerts[0]["incident_id"] == nil {
|
||||||
|
t.Fatalf("expected the alert to carry an incident_id, got %v", alerts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Alertmanager already grouped these; we adopt its answer rather than
|
||||||
|
// correlating again.
|
||||||
|
func TestWebhook_SameGroupKeyJoinsOneIncident(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
const groupKey = "{}:{alertname=\"DiskFull\"}"
|
||||||
|
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-a", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
amAlert("fp-b", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
}, groupKey)
|
||||||
|
|
||||||
|
incidents := listIncidents(t, s, "")
|
||||||
|
if len(incidents) != 1 {
|
||||||
|
t.Fatalf("expected 1 incident for one groupKey, got %d", len(incidents))
|
||||||
|
}
|
||||||
|
id := int(incidents[0]["id"].(float64))
|
||||||
|
|
||||||
|
inc := getIncident(t, s, id)
|
||||||
|
members, _ := inc["alerts"].([]any)
|
||||||
|
if len(members) != 2 {
|
||||||
|
t.Fatalf("expected 2 alerts under the incident, got %d", len(members))
|
||||||
|
}
|
||||||
|
|
||||||
|
added := 0
|
||||||
|
for _, ty := range eventTypes(timeline(t, s, id)) {
|
||||||
|
if ty == "alert_added" {
|
||||||
|
added++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if added != 2 {
|
||||||
|
t.Errorf("expected 2 alert_added events, got %d", added)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The load-bearing rule. Alertmanager re-sends firing notifications every
|
||||||
|
// repeat_interval; if those re-sends reopened incidents, resolving one by hand
|
||||||
|
// would mean nothing.
|
||||||
|
func TestWebhook_HeartbeatDoesNotReopenResolvedIncident(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
const groupKey = "{}:{alertname=\"Flapper\"}"
|
||||||
|
alert := amAlert("fp-hb", "Flapper", "firing", "2026-05-20T10:00:00Z", zeroTime, nil)
|
||||||
|
|
||||||
|
postWebhook(t, s, []map[string]any{alert}, groupKey)
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("resolve returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
|
||||||
|
// Same startsAt, same fingerprint: a re-send, not a new occurrence.
|
||||||
|
postWebhook(t, s, []map[string]any{alert}, groupKey)
|
||||||
|
|
||||||
|
if n := s.countIncidents(t); n != 1 {
|
||||||
|
t.Fatalf("expected the heartbeat to open no incident, got %d total", n)
|
||||||
|
}
|
||||||
|
if inc := getIncident(t, s, 1); inc["resolved_at"] == nil {
|
||||||
|
t.Error("expected incident 1 to stay resolved")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The alert itself is still firing and still being tracked — only the work
|
||||||
|
// item is closed.
|
||||||
|
status, _, _ := s.alertRow(t, "fp-hb")
|
||||||
|
if status != "firing" {
|
||||||
|
t.Errorf("expected the alert to still be firing, got %q", status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWebhook_NewOccurrenceOpensNewIncident(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
const groupKey = "{}:{alertname=\"Recurring\"}"
|
||||||
|
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-new", "Recurring", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
}, groupKey)
|
||||||
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
||||||
|
|
||||||
|
// A newer startsAt is a genuinely new occurrence, not a re-send.
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-new", "Recurring", "firing", "2026-05-21T09:00:00Z", zeroTime, nil),
|
||||||
|
}, groupKey)
|
||||||
|
|
||||||
|
if n := s.countIncidents(t); n != 2 {
|
||||||
|
t.Fatalf("expected a second incident for the new occurrence, got %d total", n)
|
||||||
|
}
|
||||||
|
open := listIncidents(t, s, "")
|
||||||
|
if len(open) != 1 || int(open[0]["id"].(float64)) != 2 {
|
||||||
|
t.Fatalf("expected incident 2 to be the open one, got %v", open)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The new incident starts unacknowledged: that is the point of the split.
|
||||||
|
if open[0]["acknowledged_by"] != nil {
|
||||||
|
t.Error("expected a fresh occurrence to start unacknowledged")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWebhook_ResolvedOnlyPayloadOpensNothing(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-res", "AlreadyOver", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
|
||||||
|
}, "{}:{alertname=\"AlreadyOver\"}")
|
||||||
|
|
||||||
|
if n := s.countIncidents(t); n != 0 {
|
||||||
|
t.Errorf("expected no incident from a resolved-only payload, got %d", n)
|
||||||
|
}
|
||||||
|
if status, _, _ := s.alertRow(t, "fp-res"); status != "resolved" {
|
||||||
|
t.Errorf("expected the alert itself to be stored, got %q", status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An incident that hit critical was a critical incident, even once the critical
|
||||||
|
// alert clears and only a warning is left.
|
||||||
|
func TestIncident_SeverityIsHighWaterMark(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
const groupKey = "{}:{alertname=\"Mixed\"}"
|
||||||
|
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-warn", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
||||||
|
map[string]string{"severity": "warning"}),
|
||||||
|
amAlert("fp-crit", "Mixed", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
||||||
|
map[string]string{"severity": "critical"}),
|
||||||
|
}, groupKey)
|
||||||
|
|
||||||
|
if inc := getIncident(t, s, 1); inc["severity"] != "critical" {
|
||||||
|
t.Fatalf("expected severity critical, got %v", inc["severity"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// The critical alert clears; the warning keeps the incident open.
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-crit", "Mixed", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z",
|
||||||
|
map[string]string{"severity": "critical"}),
|
||||||
|
}, groupKey)
|
||||||
|
|
||||||
|
inc := getIncident(t, s, 1)
|
||||||
|
if inc["resolved_at"] != nil {
|
||||||
|
t.Fatal("expected the incident to stay open")
|
||||||
|
}
|
||||||
|
if inc["severity"] != "critical" {
|
||||||
|
t.Errorf("expected severity to stay critical, got %v", inc["severity"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Resolution cascade
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestIncident_AllAlertsResolvedAutoResolves(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
const groupKey = "{}:{alertname=\"Pair\"}"
|
||||||
|
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-p1", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
amAlert("fp-p2", "Pair", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
}, groupKey)
|
||||||
|
|
||||||
|
// One down, one still firing: the work is not done.
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-p1", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:00:00Z", nil),
|
||||||
|
}, groupKey)
|
||||||
|
if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
|
||||||
|
t.Fatal("expected the incident to stay open while an alert is firing")
|
||||||
|
}
|
||||||
|
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-p2", "Pair", "resolved", "2026-05-20T10:00:00Z", "2026-05-20T11:30:00Z", nil),
|
||||||
|
}, groupKey)
|
||||||
|
|
||||||
|
inc := getIncident(t, s, 1)
|
||||||
|
if inc["status"] != "resolved" {
|
||||||
|
t.Errorf("expected status resolved, got %v", inc["status"])
|
||||||
|
}
|
||||||
|
if inc["resolution_source"] != "alerts" {
|
||||||
|
t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expiry is inference, not observation, but it still has to close the work item
|
||||||
|
// — otherwise a lost resolved notification leaves an incident open forever.
|
||||||
|
func TestExpiry_CascadesToIncidentResolution(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postAlert(t, s, "fp-exp", "firing", time.Now().Add(-24*time.Hour).Format(time.RFC3339), zeroTime)
|
||||||
|
|
||||||
|
s.exec(t, "UPDATE alerts SET received_at = ? WHERE fingerprint = 'fp-exp'",
|
||||||
|
time.Now().Add(-10*time.Hour).Unix())
|
||||||
|
sweep(t, s, 6*time.Hour)
|
||||||
|
|
||||||
|
inc := getIncident(t, s, 1)
|
||||||
|
if inc["status"] != "resolved" {
|
||||||
|
t.Errorf("expected the incident to resolve after expiry, got %v", inc["status"])
|
||||||
|
}
|
||||||
|
if inc["resolution_source"] != "alerts" {
|
||||||
|
t.Errorf("expected resolution_source alerts, got %v", inc["resolution_source"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// The expiry is recorded against the alert, not the incident.
|
||||||
|
if _, source, _ := s.alertRow(t, "fp-exp"); source == nil || *source != "expiry" {
|
||||||
|
t.Errorf("expected the alert's resolution_source to stay expiry, got %v", source)
|
||||||
|
}
|
||||||
|
if types := eventTypes(timeline(t, s, 1)); !contains(types, "alert_resolved") {
|
||||||
|
t.Errorf("expected an alert_resolved event on the timeline, got %v", types)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Workflow actions
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestIncident_Acknowledge(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-ack", "X", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("acknowledge returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
var inc map[string]any
|
||||||
|
decode(t, resp, &inc)
|
||||||
|
if inc["acknowledged_by"] == nil {
|
||||||
|
t.Error("expected acknowledged_by to be set")
|
||||||
|
}
|
||||||
|
if inc["status"] != "acknowledged" {
|
||||||
|
t.Errorf("expected status acknowledged, got %v", inc["status"])
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/acknowledge", nil)
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusNoContent {
|
||||||
|
t.Fatalf("unacknowledge returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
|
||||||
|
t.Errorf("expected status back to triggered, got %v", inc["status"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncident_ManualResolveIsTerminal(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-term", "Terminal", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("first resolve returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
var inc map[string]any
|
||||||
|
decode(t, resp, &inc)
|
||||||
|
if inc["resolution_source"] != "manual" {
|
||||||
|
t.Errorf("expected resolution_source manual, got %v", inc["resolution_source"])
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusConflict {
|
||||||
|
t.Errorf("expected 409 on re-resolve, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Acknowledging a closed incident is equally meaningless.
|
||||||
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusConflict {
|
||||||
|
t.Errorf("expected 409 acknowledging a resolved incident, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncident_SnoozeHiddenFromDefaultList(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-snz", "Noisy", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze", map[string]string{"duration": "2h"})
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("snooze returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
var inc map[string]any
|
||||||
|
decode(t, resp, &inc)
|
||||||
|
if inc["snoozed_until"] == nil {
|
||||||
|
t.Error("expected snoozed_until to be set")
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := listIncidents(t, s, ""); len(got) != 0 {
|
||||||
|
t.Errorf("expected the snoozed incident to be hidden, got %d", len(got))
|
||||||
|
}
|
||||||
|
if got := listIncidents(t, s, "?snoozed=true"); len(got) != 1 {
|
||||||
|
t.Errorf("expected snoozed=true to show it, got %d", len(got))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A snooze is not a resolution: the incident is still open work.
|
||||||
|
if inc := getIncident(t, s, 1); inc["resolved_at"] != nil {
|
||||||
|
t.Error("expected a snoozed incident to stay open")
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/snooze", nil)
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusNoContent {
|
||||||
|
t.Fatalf("unsnooze returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
if got := listIncidents(t, s, ""); len(got) != 1 {
|
||||||
|
t.Errorf("expected the incident back in the default list, got %d", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncident_SnoozeRejectsPastDeadline(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-past", "Past", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze",
|
||||||
|
map[string]string{"until": time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)})
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusBadRequest {
|
||||||
|
t.Errorf("expected 400 for a snooze in the past, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The schedule stops being decorative here: it is read at trigger time.
|
||||||
|
func TestIncident_AutoAssignedToCurrentOnCall(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
|
||||||
|
today := time.Now().UTC().Format("2006-01-02")
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/schedule",
|
||||||
|
map[string]any{"user_id": 1, "dates": []string{today}})
|
||||||
|
if resp.StatusCode != http.StatusCreated {
|
||||||
|
t.Fatalf("schedule assignment returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-oncall", "PageMe", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
inc := getIncident(t, s, 1)
|
||||||
|
if inc["assigned_to"] != "admin" {
|
||||||
|
t.Errorf("expected the incident assigned to today's on-call, got %v", inc["assigned_to"])
|
||||||
|
}
|
||||||
|
if types := eventTypes(timeline(t, s, 1)); !contains(types, "assigned") {
|
||||||
|
t.Errorf("expected an assigned event, got %v", types)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncident_AssignToUser(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-asg", "Assignable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
s.req(t, http.MethodPost, "/api/users",
|
||||||
|
map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 2})
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("assign returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
var inc map[string]any
|
||||||
|
decode(t, resp, &inc)
|
||||||
|
if inc["assigned_to"] != "alice" {
|
||||||
|
t.Errorf("expected assigned_to alice, got %v", inc["assigned_to"])
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = s.req(t, http.MethodPost, "/api/incidents/1/assign", map[string]any{"user_id": 99})
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusNotFound {
|
||||||
|
t.Errorf("expected 404 assigning an unknown user, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Timeline and notes
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestIncident_TimelineOrdering(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-tl", "Storyline", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
|
||||||
|
s.req(t, http.MethodPost, "/api/incidents/1/notes",
|
||||||
|
map[string]string{"content": "looking into it"}).Body.Close()
|
||||||
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
||||||
|
|
||||||
|
events := timeline(t, s, 1)
|
||||||
|
want := []string{"triggered", "alert_added", "acknowledged", "note", "resolved"}
|
||||||
|
got := eventTypes(events)
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("expected timeline %v, got %v", want, got)
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] {
|
||||||
|
t.Fatalf("expected timeline %v, got %v", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The note carries its author; system events do not.
|
||||||
|
for _, e := range events {
|
||||||
|
if e["type"] == "note" {
|
||||||
|
if e["username"] != "admin" || e["detail"] != "looking into it" {
|
||||||
|
t.Errorf("unexpected note event: %v", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if e["type"] == "triggered" && e["username"] != nil {
|
||||||
|
t.Errorf("expected the triggered event to have no author, got %v", e["username"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncident_NoteDeleteOwnOnly(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-note", "Y", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
s.req(t, http.MethodPost, "/api/users",
|
||||||
|
map[string]string{"username": "alice", "email": "alice@test.com"}).Body.Close()
|
||||||
|
var keyData map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodPost, "/api/users/2/api-keys",
|
||||||
|
map[string]string{"name": "alice-key"}), &keyData)
|
||||||
|
aliceKey := keyData["key"].(string)
|
||||||
|
|
||||||
|
var note map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodPost, "/api/incidents/1/notes",
|
||||||
|
map[string]string{"content": "admin note"}), ¬e)
|
||||||
|
noteID := int(note["id"].(float64))
|
||||||
|
|
||||||
|
// Alice cannot delete admin's note.
|
||||||
|
req, _ := http.NewRequest(http.MethodDelete,
|
||||||
|
fmt.Sprintf("%s/api/incidents/1/notes/%d", s.URL, noteID), nil)
|
||||||
|
req.Header.Set("Authorization", "Bearer "+aliceKey)
|
||||||
|
resp, _ := http.DefaultClient.Do(req)
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusNotFound {
|
||||||
|
t.Errorf("expected 404 deleting another user's note, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", noteID), nil)
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusNoContent {
|
||||||
|
t.Errorf("expected 204 deleting own note, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only notes are deletable — the rest of the timeline is what happened.
|
||||||
|
func TestIncident_CannotDeleteSystemEvent(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-sys", "System", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
events := timeline(t, s, 1)
|
||||||
|
id := int(events[0]["id"].(float64))
|
||||||
|
resp := s.req(t, http.MethodDelete, fmt.Sprintf("/api/incidents/1/notes/%d", id), nil)
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusNotFound {
|
||||||
|
t.Errorf("expected 404 deleting a system event, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Archive
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestIncident_ArchiveRoundTrip(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-arc", "Archivable", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/archive", nil)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("archive returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
var inc map[string]any
|
||||||
|
decode(t, resp, &inc)
|
||||||
|
if inc["archived_at"] == nil {
|
||||||
|
t.Error("expected archived_at to be set")
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := listIncidents(t, s, "?status=resolved"); len(got) != 0 {
|
||||||
|
t.Errorf("expected the archived incident to be hidden, got %d", len(got))
|
||||||
|
}
|
||||||
|
if got := listIncidents(t, s, "?status=resolved&archived=true"); len(got) != 1 {
|
||||||
|
t.Errorf("expected archived=true to show it, got %d", len(got))
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = s.req(t, http.MethodDelete, "/api/incidents/1/archive", nil)
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusNoContent {
|
||||||
|
t.Fatalf("unarchive returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
if got := listIncidents(t, s, "?status=resolved"); len(got) != 1 {
|
||||||
|
t.Errorf("expected the incident back, got %d", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSweeper_ArchivesResolvedIncidents(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-swp", "Old", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
||||||
|
|
||||||
|
s.exec(t, "UPDATE incidents SET resolved_at = ? WHERE id = 1",
|
||||||
|
time.Now().Add(-30*24*time.Hour).Unix())
|
||||||
|
api.Sweep(context.Background(), s.db, 7*24*time.Hour, 6*time.Hour, s.deadman, s.notify)
|
||||||
|
|
||||||
|
if inc := getIncident(t, s, 1); inc["archived_at"] == nil {
|
||||||
|
t.Error("expected the sweeper to archive a long-resolved incident")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Stats
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestStats_Incidents(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-s1", "One", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
}, "g1")
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-s2", "Two", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
}, "g2")
|
||||||
|
s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil).Body.Close()
|
||||||
|
s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil).Body.Close()
|
||||||
|
|
||||||
|
var stats map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
|
||||||
|
|
||||||
|
if stats["total"].(float64) != 2 {
|
||||||
|
t.Errorf("expected total 2, got %v", stats["total"])
|
||||||
|
}
|
||||||
|
if stats["resolved"].(float64) != 1 {
|
||||||
|
t.Errorf("expected resolved 1, got %v", stats["resolved"])
|
||||||
|
}
|
||||||
|
if stats["triggered"].(float64) != 1 {
|
||||||
|
t.Errorf("expected triggered 1, got %v", stats["triggered"])
|
||||||
|
}
|
||||||
|
// One incident has been acknowledged and resolved, so both averages exist.
|
||||||
|
if stats["mtta_seconds"] == nil || stats["mttr_seconds"] == nil {
|
||||||
|
t.Errorf("expected mtta and mttr to be computable, got %v", stats)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An empty window is a report of zero, not a failure. SUM over no rows is NULL
|
||||||
|
// in SQLite, which used to come back as a 500 the moment every incident was
|
||||||
|
// archived — the state a quiet installation settles into.
|
||||||
|
func TestStats_IncidentsEmptyWindowIsZeroNotAnError(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
|
||||||
|
// No incidents at all.
|
||||||
|
resp := s.req(t, http.MethodGet, "/api/stats/incidents", nil)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
resp.Body.Close()
|
||||||
|
t.Fatalf("expected 200 on an empty database, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
var stats map[string]any
|
||||||
|
decode(t, resp, &stats)
|
||||||
|
for _, k := range []string{"total", "triggered", "acknowledged", "resolved"} {
|
||||||
|
if stats[k].(float64) != 0 {
|
||||||
|
t.Errorf("expected %s 0, got %v", k, stats[k])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And with every incident archived out of the window.
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-s4", "Gone", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
s.req(t, http.MethodPost, "/api/incidents/1/archive", nil).Body.Close()
|
||||||
|
|
||||||
|
resp = s.req(t, http.MethodGet, "/api/stats/incidents", nil)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
resp.Body.Close()
|
||||||
|
t.Fatalf("expected 200 when every incident is archived, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
stats = nil
|
||||||
|
decode(t, resp, &stats)
|
||||||
|
if stats["total"].(float64) != 0 {
|
||||||
|
t.Errorf("expected total 0, got %v", stats["total"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The alert stats share the same aggregate, and the same empty-window trap.
|
||||||
|
func TestStats_AlertsEmptyWindowIsZeroNotAnError(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
resp := s.req(t, http.MethodGet, "/api/stats/alerts", nil)
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
resp.Body.Close()
|
||||||
|
t.Fatalf("expected 200 on an empty database, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
var stats map[string]any
|
||||||
|
decode(t, resp, &stats)
|
||||||
|
for _, k := range []string{"total", "firing", "resolved"} {
|
||||||
|
if stats[k].(float64) != 0 {
|
||||||
|
t.Errorf("expected %s 0, got %v", k, stats[k])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing acknowledged yet means "no data", which is not the same claim as zero.
|
||||||
|
func TestStats_IncidentsNullMTTAWhenNothingAcknowledged(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-s3", "Untouched", "firing", "2026-05-20T10:00:00Z", zeroTime, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
var stats map[string]any
|
||||||
|
decode(t, s.req(t, http.MethodGet, "/api/stats/incidents", nil), &stats)
|
||||||
|
if stats["mtta_seconds"] != nil {
|
||||||
|
t.Errorf("expected mtta_seconds null, got %v", stats["mtta_seconds"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Migration backfill
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// An upgrade must not drop the acknowledgements and comments people already
|
||||||
|
// have, so 008 is replayed here over a database left at 007.
|
||||||
|
func TestMigration_BackfillCarriesAckAndComments(t *testing.T) {
|
||||||
|
database, err := db.Open(":memory:")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open db: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { database.Close() })
|
||||||
|
|
||||||
|
files, err := filepath.Glob("../db/migrations/*.sql")
|
||||||
|
if err != nil || len(files) == 0 {
|
||||||
|
t.Fatalf("find migrations: %v", err)
|
||||||
|
}
|
||||||
|
sort.Strings(files)
|
||||||
|
|
||||||
|
var incidentsMigration string
|
||||||
|
for _, f := range files {
|
||||||
|
if filepath.Base(f) >= "008" {
|
||||||
|
incidentsMigration = f
|
||||||
|
break
|
||||||
|
}
|
||||||
|
data, err := os.ReadFile(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", f, err)
|
||||||
|
}
|
||||||
|
if _, err := database.Exec(string(data)); err != nil {
|
||||||
|
t.Fatalf("apply %s: %v", f, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if incidentsMigration == "" {
|
||||||
|
t.Fatal("008 migration not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A database as it would look on the old schema: an acknowledged firing
|
||||||
|
// alert with a comment on it.
|
||||||
|
if _, err := database.Exec(`
|
||||||
|
INSERT INTO users (id, username, email) VALUES (1, 'admin', 'admin@test.com');
|
||||||
|
INSERT INTO alerts (id, fingerprint, name, status, labels, annotations,
|
||||||
|
starts_at, received_at, acknowledged_by, acknowledged_at)
|
||||||
|
VALUES (1, 'legacy-fp', 'LegacyAlert', 'firing',
|
||||||
|
'{"severity":"warning"}', '{}', 1000, 1000, 1, 1500);
|
||||||
|
INSERT INTO alert_comments (alert_id, user_id, content, created_at)
|
||||||
|
VALUES (1, 1, 'legacy comment', 1600);`); err != nil {
|
||||||
|
t.Fatalf("seed pre-008 data: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := os.ReadFile(incidentsMigration)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read 008: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := database.Exec(string(data)); err != nil {
|
||||||
|
t.Fatalf("apply 008: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var status, groupKey string
|
||||||
|
var ackBy int64
|
||||||
|
var severity string
|
||||||
|
if err := database.QueryRow(
|
||||||
|
"SELECT status, group_key, acknowledged_by, severity FROM incidents WHERE id = 1",
|
||||||
|
).Scan(&status, &groupKey, &ackBy, &severity); err != nil {
|
||||||
|
t.Fatalf("read backfilled incident: %v", err)
|
||||||
|
}
|
||||||
|
if status != "acknowledged" {
|
||||||
|
t.Errorf("expected the ack to carry over as status, got %q", status)
|
||||||
|
}
|
||||||
|
if groupKey != "backfill:legacy-fp" {
|
||||||
|
t.Errorf("unexpected group_key %q", groupKey)
|
||||||
|
}
|
||||||
|
if ackBy != 1 {
|
||||||
|
t.Errorf("expected acknowledged_by 1, got %d", ackBy)
|
||||||
|
}
|
||||||
|
if severity != "warning" {
|
||||||
|
t.Errorf("expected severity carried from labels, got %q", severity)
|
||||||
|
}
|
||||||
|
|
||||||
|
var notes int
|
||||||
|
if err := database.QueryRow(
|
||||||
|
"SELECT COUNT(*) FROM incident_events WHERE type = 'note' AND detail = 'legacy comment'",
|
||||||
|
).Scan(¬es); err != nil {
|
||||||
|
t.Fatalf("count notes: %v", err)
|
||||||
|
}
|
||||||
|
if notes != 1 {
|
||||||
|
t.Errorf("expected the comment to become a note, got %d", notes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the columns that caused the ack-survives-a-re-fire bug are gone.
|
||||||
|
if _, err := database.Exec("SELECT acknowledged_by FROM alerts"); err == nil {
|
||||||
|
t.Error("expected alerts.acknowledged_by to be dropped")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(haystack []string, needle string) bool {
|
||||||
|
for _, s := range haystack {
|
||||||
|
if s == needle {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
+107
-27
@@ -9,55 +9,135 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/yeniklas/terdut-server/internal/models"
|
"git.ryuvia.com/niklas/terdut-server/internal/models"
|
||||||
)
|
)
|
||||||
|
|
||||||
type contextKey string
|
type contextKey string
|
||||||
|
|
||||||
const ctxUser contextKey = "user"
|
const (
|
||||||
|
ctxUser contextKey = "user"
|
||||||
|
ctxSession contextKey = "session"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AuthMiddleware accepts either of the two credentials the server issues: an
|
||||||
|
// API key in an Authorization header (the TUI, scripts) or a session cookie
|
||||||
|
// (the web UI). A request carrying a Bearer header is judged on that alone and
|
||||||
|
// never falls back to the cookie.
|
||||||
|
//
|
||||||
|
// Only the cookie needs a CSRF guard. A browser attaches it to requests other
|
||||||
|
// sites make, whereas an Authorization header is only ever set by the client
|
||||||
|
// that holds the key.
|
||||||
func AuthMiddleware(db *sql.DB) func(http.Handler) http.Handler {
|
func AuthMiddleware(db *sql.DB) func(http.Handler) http.Handler {
|
||||||
|
crossOrigin := http.NewCrossOriginProtection()
|
||||||
|
|
||||||
return func(next http.Handler) http.Handler {
|
return func(next http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
|
if header := r.Header.Get("Authorization"); header != "" {
|
||||||
if !ok || token == "" {
|
token, ok := strings.CutPrefix(header, "Bearer ")
|
||||||
respond(w, http.StatusUnauthorized, errResp("unauthorized"))
|
if !ok || token == "" {
|
||||||
|
respond(w, http.StatusUnauthorized, errResp("unauthorized"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
userID, ok := apiKeyUser(r.Context(), db, token)
|
||||||
|
if !ok {
|
||||||
|
respond(w, http.StatusUnauthorized, errResp("unauthorized"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
serveAs(w, r, next, db, userID, 0)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
h := sha256.Sum256([]byte(token))
|
c, err := r.Cookie(sessionCookie)
|
||||||
hash := hex.EncodeToString(h[:])
|
if err != nil || c.Value == "" {
|
||||||
|
|
||||||
var keyID, userID int64
|
|
||||||
err := db.QueryRowContext(r.Context(),
|
|
||||||
"SELECT id, user_id FROM api_keys WHERE key_hash = ?", hash,
|
|
||||||
).Scan(&keyID, &userID)
|
|
||||||
if err != nil {
|
|
||||||
respond(w, http.StatusUnauthorized, errResp("unauthorized"))
|
respond(w, http.StatusUnauthorized, errResp("unauthorized"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
sessionID, userID, ok := sessionUser(r.Context(), db, c.Value)
|
||||||
// best-effort; don't fail the request if this update fails
|
if !ok {
|
||||||
db.ExecContext(r.Context(),
|
|
||||||
"UPDATE api_keys SET last_used_at = ? WHERE id = ?",
|
|
||||||
time.Now().Unix(), keyID)
|
|
||||||
|
|
||||||
var u models.User
|
|
||||||
var createdUnix int64
|
|
||||||
if err := db.QueryRowContext(r.Context(),
|
|
||||||
"SELECT id, username, email, created_at FROM users WHERE id = ?", userID,
|
|
||||||
).Scan(&u.ID, &u.Username, &u.Email, &createdUnix); err != nil {
|
|
||||||
respond(w, http.StatusUnauthorized, errResp("unauthorized"))
|
respond(w, http.StatusUnauthorized, errResp("unauthorized"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
u.CreatedAt = time.Unix(createdUnix, 0).UTC()
|
if err := crossOrigin.Check(r); err != nil {
|
||||||
|
respond(w, http.StatusForbidden, errResp("cross-origin request rejected"))
|
||||||
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), ctxUser, u)))
|
return
|
||||||
|
}
|
||||||
|
serveAs(w, r, next, db, userID, sessionID)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// apiKeyUser resolves an API key to its user and stamps its last use.
|
||||||
|
func apiKeyUser(ctx context.Context, db *sql.DB, token string) (int64, bool) {
|
||||||
|
var keyID, userID int64
|
||||||
|
err := db.QueryRowContext(ctx,
|
||||||
|
"SELECT id, user_id FROM api_keys WHERE key_hash = ?", hashToken(token),
|
||||||
|
).Scan(&keyID, &userID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// best-effort; don't fail the request if this update fails
|
||||||
|
db.ExecContext(ctx,
|
||||||
|
"UPDATE api_keys SET last_used_at = ? WHERE id = ?",
|
||||||
|
time.Now().Unix(), keyID)
|
||||||
|
return userID, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// sessionUser resolves a session token to its session and user. The expiry
|
||||||
|
// slides forward with use, but at most once per sessionTouchEvery, so a page
|
||||||
|
// that polls does not write to the database on every request.
|
||||||
|
func sessionUser(ctx context.Context, db *sql.DB, token string) (sessionID, userID int64, ok bool) {
|
||||||
|
now := time.Now()
|
||||||
|
var lastSeen int64
|
||||||
|
err := db.QueryRowContext(ctx, `
|
||||||
|
SELECT id, user_id, last_seen_at FROM sessions
|
||||||
|
WHERE token_hash = ? AND expires_at > ?`,
|
||||||
|
hashToken(token), now.Unix()).Scan(&sessionID, &userID, &lastSeen)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
if now.Sub(time.Unix(lastSeen, 0)) > sessionTouchEvery {
|
||||||
|
db.ExecContext(ctx,
|
||||||
|
"UPDATE sessions SET last_seen_at = ?, expires_at = ? WHERE id = ?",
|
||||||
|
now.Unix(), now.Add(sessionTTL).Unix(), sessionID)
|
||||||
|
}
|
||||||
|
return sessionID, userID, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// serveAs loads the user and hands the request on with it in the context.
|
||||||
|
// sessionID is zero for API-key requests.
|
||||||
|
func serveAs(w http.ResponseWriter, r *http.Request, next http.Handler, db *sql.DB, userID, sessionID int64) {
|
||||||
|
var u models.User
|
||||||
|
var createdUnix int64
|
||||||
|
if err := db.QueryRowContext(r.Context(),
|
||||||
|
"SELECT id, username, email, created_at FROM users WHERE id = ?", userID,
|
||||||
|
).Scan(&u.ID, &u.Username, &u.Email, &createdUnix); err != nil {
|
||||||
|
respond(w, http.StatusUnauthorized, errResp("unauthorized"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
u.CreatedAt = time.Unix(createdUnix, 0).UTC()
|
||||||
|
|
||||||
|
ctx := context.WithValue(r.Context(), ctxUser, u)
|
||||||
|
if sessionID != 0 {
|
||||||
|
ctx = context.WithValue(ctx, ctxSession, sessionID)
|
||||||
|
}
|
||||||
|
next.ServeHTTP(w, r.WithContext(ctx))
|
||||||
|
}
|
||||||
|
|
||||||
|
func hashToken(token string) string {
|
||||||
|
h := sha256.Sum256([]byte(token))
|
||||||
|
return hex.EncodeToString(h[:])
|
||||||
|
}
|
||||||
|
|
||||||
func userFromContext(ctx context.Context) (models.User, bool) {
|
func userFromContext(ctx context.Context) (models.User, bool) {
|
||||||
u, ok := ctx.Value(ctxUser).(models.User)
|
u, ok := ctx.Value(ctxUser).(models.User)
|
||||||
return u, ok
|
return u, ok
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sessionFromContext returns the id of the session a request was authenticated
|
||||||
|
// with, or false for an API-key request.
|
||||||
|
func sessionFromContext(ctx context.Context) (int64, bool) {
|
||||||
|
id, ok := ctx.Value(ctxSession).(int64)
|
||||||
|
return id, ok
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,565 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// notifyInterval is how often the notifier looks for work. The archiver's
|
||||||
|
// 15 minute tick is far too coarse for something that has to wake a person.
|
||||||
|
notifyInterval = 30 * time.Second
|
||||||
|
|
||||||
|
// notifyRetryBase and notifyRetryMax bound the delivery backoff. ntfy being
|
||||||
|
// briefly unreachable should not lose the page.
|
||||||
|
notifyRetryBase = 30 * time.Second
|
||||||
|
notifyRetryMax = 15 * time.Minute
|
||||||
|
|
||||||
|
// notifyMaxAttempts stops a permanently undeliverable row from being retried
|
||||||
|
// forever. It keeps last_error so the reason survives.
|
||||||
|
notifyMaxAttempts = 8
|
||||||
|
|
||||||
|
// notifyBatch caps one delivery pass, so a large backlog cannot hold the
|
||||||
|
// single database connection for an unbounded stretch.
|
||||||
|
notifyBatch = 100
|
||||||
|
|
||||||
|
// ackTokenTTL is how long the Acknowledge button in a notification keeps
|
||||||
|
// working. Past this the notification is stale enough that the responder
|
||||||
|
// should look at the incident rather than blind-acknowledge it.
|
||||||
|
ackTokenTTL = 24 * time.Hour
|
||||||
|
)
|
||||||
|
|
||||||
|
// Notification kinds, recording why a push was sent.
|
||||||
|
const (
|
||||||
|
notifyTriggered = "triggered"
|
||||||
|
notifyReminder = "reminder"
|
||||||
|
notifyResolved = "resolved"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Timeline event types the notifier writes, so an incident's history says who
|
||||||
|
// was paged and whether the page landed. Written from the delivery result
|
||||||
|
// rather than at enqueue: a queued notification is an intention, and claiming
|
||||||
|
// somebody was told before ntfy accepted it would be a lie the timeline keeps.
|
||||||
|
//
|
||||||
|
// The topic is deliberately absent from both. It is a shared secret with the
|
||||||
|
// ntfy server — anyone holding it can publish to it — and the timeline is
|
||||||
|
// readable by every API key.
|
||||||
|
const (
|
||||||
|
eventNotified = "notified"
|
||||||
|
eventNotifyFailed = "notify_failed"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NotifyConfig is everything the notifier needs to reach ntfy and to build URLs
|
||||||
|
// a phone can follow back to this server.
|
||||||
|
type NotifyConfig struct {
|
||||||
|
// BaseURL is the ntfy server. Empty disables notifications entirely: no
|
||||||
|
// goroutine, and nothing is ever enqueued.
|
||||||
|
BaseURL string
|
||||||
|
|
||||||
|
// Token is an optional bearer token for an access-controlled ntfy.
|
||||||
|
Token string
|
||||||
|
|
||||||
|
// FallbackTopic receives incidents that open with nobody on call. Those
|
||||||
|
// notifications carry no Acknowledge button — there is no user to attribute
|
||||||
|
// the acknowledgement to, and putting one on a shared topic would let any
|
||||||
|
// subscriber acknowledge as somebody else.
|
||||||
|
FallbackTopic string
|
||||||
|
|
||||||
|
// PublicURL is the base URL a phone uses to reach this server, for the
|
||||||
|
// notification's click target and its Acknowledge action. Without it a
|
||||||
|
// notification is informational only.
|
||||||
|
PublicURL string
|
||||||
|
|
||||||
|
// RepeatEvery is how long an incident may sit unacknowledged before it is
|
||||||
|
// notified again. Zero disables reminders.
|
||||||
|
RepeatEvery time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// enabled reports whether notifications are configured at all.
|
||||||
|
func (c NotifyConfig) enabled() bool { return c.BaseURL != "" }
|
||||||
|
|
||||||
|
// notifyClient is shared: a page is small and infrequent, and the timeout is
|
||||||
|
// what keeps a hung ntfy from stalling the delivery pass.
|
||||||
|
var notifyClient = &http.Client{Timeout: 10 * time.Second}
|
||||||
|
|
||||||
|
// StartNotifier delivers queued notifications until ctx is cancelled, starting
|
||||||
|
// with an immediate pass so a restart flushes whatever the last one left behind.
|
||||||
|
func StartNotifier(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
|
||||||
|
if !cfg.enabled() {
|
||||||
|
log.Print("notifier: disabled (no ntfy URL configured)")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
log.Printf("notifier: publishing to %s", cfg.BaseURL)
|
||||||
|
|
||||||
|
ticker := time.NewTicker(notifyInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
NotifySweep(ctx, db, cfg)
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
NotifySweep(ctx, db, cfg)
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NotifySweep runs a single pass: queue reminders for incidents nobody has
|
||||||
|
// picked up, then deliver everything that is due. Reminders are queued first so
|
||||||
|
// a freshly due one goes out in the same pass rather than a tick later.
|
||||||
|
// Exported so tests can drive a pass without waiting on the ticker.
|
||||||
|
func NotifySweep(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
|
||||||
|
enqueueReminders(ctx, db, cfg)
|
||||||
|
deliverPending(ctx, db, cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// enqueueReminders re-notifies incidents that are still sitting untouched.
|
||||||
|
//
|
||||||
|
// The stop conditions are the incident states that already mean "somebody has
|
||||||
|
// this": acknowledged, snoozed, resolved, archived. Snooze in particular is the
|
||||||
|
// mute button — a deliberate "not now" that should not keep buzzing — which is
|
||||||
|
// why there is no separate reminder cap.
|
||||||
|
//
|
||||||
|
// The previous notification must have actually been sent before another is
|
||||||
|
// queued, so an ntfy outage produces a retry backlog rather than a reminder
|
||||||
|
// backlog that all lands at once when it comes back.
|
||||||
|
func enqueueReminders(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
|
||||||
|
if cfg.RepeatEvery <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
type due struct {
|
||||||
|
incidentID int64
|
||||||
|
userID *int64
|
||||||
|
topic string
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := db.QueryContext(ctx, `
|
||||||
|
SELECT n.incident_id, n.user_id, n.topic
|
||||||
|
FROM notifications n
|
||||||
|
JOIN incidents i ON i.id = n.incident_id
|
||||||
|
WHERE n.id = (SELECT MAX(id) FROM notifications WHERE incident_id = n.incident_id)
|
||||||
|
AND n.sent_at IS NOT NULL
|
||||||
|
AND n.created_at <= ?
|
||||||
|
AND i.resolved_at IS NULL
|
||||||
|
AND i.archived_at IS NULL
|
||||||
|
AND i.status = 'triggered'
|
||||||
|
AND (i.snoozed_until IS NULL OR i.snoozed_until <= ?)`,
|
||||||
|
now.Add(-cfg.RepeatEvery).Unix(), now.Unix())
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("notifier: find reminders: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collected before inserting: the pool is limited to a single connection, so
|
||||||
|
// an open cursor would block the writes behind it.
|
||||||
|
var pending []due
|
||||||
|
for rows.Next() {
|
||||||
|
var d due
|
||||||
|
if err := rows.Scan(&d.incidentID, &d.userID, &d.topic); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
log.Printf("notifier: scan reminder: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pending = append(pending, d)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
log.Printf("notifier: find reminders: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
|
||||||
|
for _, d := range pending {
|
||||||
|
if err := enqueueNotification(ctx, db, d.incidentID, d.userID, d.topic, notifyReminder); err != nil {
|
||||||
|
log.Printf("notifier: queue reminder for incident %d: %v", d.incidentID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(pending) > 0 {
|
||||||
|
log.Printf("notifier: queued %d reminder(s)", len(pending))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// outboxRow is one queued notification, read before any HTTP happens.
|
||||||
|
type outboxRow struct {
|
||||||
|
id int64
|
||||||
|
incidentID int64
|
||||||
|
userID *int64
|
||||||
|
topic string
|
||||||
|
kind string
|
||||||
|
attempts int
|
||||||
|
}
|
||||||
|
|
||||||
|
// deliverPending sends everything that is due and records the outcome.
|
||||||
|
func deliverPending(ctx context.Context, db *sql.DB, cfg NotifyConfig) {
|
||||||
|
batch, err := pendingNotifications(ctx, db)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("notifier: find pending: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sent := 0
|
||||||
|
for _, n := range batch {
|
||||||
|
if err := deliver(ctx, db, cfg, n); err != nil {
|
||||||
|
log.Printf("notifier: deliver %d (incident %d): %v", n.id, n.incidentID, err)
|
||||||
|
markFailed(ctx, db, n, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := db.ExecContext(ctx,
|
||||||
|
"UPDATE notifications SET sent_at = ?, attempts = attempts + 1, last_error = NULL WHERE id = ?",
|
||||||
|
time.Now().Unix(), n.id); err != nil {
|
||||||
|
log.Printf("notifier: mark sent %d: %v", n.id, err)
|
||||||
|
}
|
||||||
|
// Logged, not returned: the page has already gone out, and treating a
|
||||||
|
// failed timeline write as a failed delivery would send it again.
|
||||||
|
if err := logEvent(ctx, db, n.incidentID, eventNotified, n.userID, nil, &n.kind); err != nil {
|
||||||
|
log.Printf("notifier: log delivery of %d: %v", n.id, err)
|
||||||
|
}
|
||||||
|
sent++
|
||||||
|
}
|
||||||
|
if sent > 0 {
|
||||||
|
log.Printf("notifier: delivered %d notification(s)", sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pendingNotifications reads the due rows and closes the cursor before the
|
||||||
|
// caller writes, for the same single-connection reason as staleAlertIDs.
|
||||||
|
func pendingNotifications(ctx context.Context, db *sql.DB) ([]outboxRow, error) {
|
||||||
|
rows, err := db.QueryContext(ctx, `
|
||||||
|
SELECT id, incident_id, user_id, topic, kind, attempts
|
||||||
|
FROM notifications
|
||||||
|
WHERE sent_at IS NULL
|
||||||
|
AND send_after <= ?
|
||||||
|
AND attempts < ?
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT ?`, time.Now().Unix(), notifyMaxAttempts, notifyBatch)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var batch []outboxRow
|
||||||
|
for rows.Next() {
|
||||||
|
var n outboxRow
|
||||||
|
if err := rows.Scan(&n.id, &n.incidentID, &n.userID, &n.topic, &n.kind, &n.attempts); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
batch = append(batch, n)
|
||||||
|
}
|
||||||
|
return batch, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// markFailed bumps the attempt count and pushes the row out to its next retry.
|
||||||
|
//
|
||||||
|
// The attempt that exhausts the budget also writes a timeline event. Without it
|
||||||
|
// a page that never landed leaves the incident's history identical to one that
|
||||||
|
// did, which is the failure most worth seeing: nobody was told, and nothing
|
||||||
|
// says so.
|
||||||
|
func markFailed(ctx context.Context, db *sql.DB, n outboxRow, cause error) {
|
||||||
|
next := time.Now().Add(retryDelay(n.attempts)).Unix()
|
||||||
|
if _, err := db.ExecContext(ctx,
|
||||||
|
"UPDATE notifications SET attempts = attempts + 1, send_after = ?, last_error = ? WHERE id = ?",
|
||||||
|
next, cause.Error(), n.id); err != nil {
|
||||||
|
log.Printf("notifier: mark failed %d: %v", n.id, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n.attempts+1 < notifyMaxAttempts {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
detail := fmt.Sprintf("%s: %s", n.kind, cause)
|
||||||
|
if err := logEvent(ctx, db, n.incidentID, eventNotifyFailed, n.userID, nil, &detail); err != nil {
|
||||||
|
log.Printf("notifier: log failure of %d: %v", n.id, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryDelay doubles the wait per attempt, up to notifyRetryMax.
|
||||||
|
func retryDelay(attempts int) time.Duration {
|
||||||
|
d := notifyRetryBase << attempts
|
||||||
|
if d > notifyRetryMax || d <= 0 {
|
||||||
|
return notifyRetryMax
|
||||||
|
}
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// deliver renders one notification against the incident's *current* state and
|
||||||
|
// publishes it. Rendering happens here rather than at enqueue time so a message
|
||||||
|
// that waited in the queue while its incident escalated goes out at the
|
||||||
|
// severity the incident has now.
|
||||||
|
func deliver(ctx context.Context, db *sql.DB, cfg NotifyConfig, n outboxRow) error {
|
||||||
|
inc, err := fetchIncident(ctx, db, n.incidentID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("load incident: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var firing int
|
||||||
|
if err := db.QueryRowContext(ctx, `
|
||||||
|
SELECT COUNT(*)
|
||||||
|
FROM incident_alerts ia
|
||||||
|
JOIN alerts a ON a.id = ia.alert_id
|
||||||
|
WHERE ia.incident_id = ? AND a.status = 'firing'`, n.incidentID).Scan(&firing); err != nil {
|
||||||
|
return fmt.Errorf("count firing: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
msg := renderNotification(inc, n, firing, cfg)
|
||||||
|
|
||||||
|
// An Acknowledge button needs both a user to attribute the acknowledgement
|
||||||
|
// to and a URL the phone can reach. Minted per delivery, so every push
|
||||||
|
// carries its own short-lived token rather than reusing one.
|
||||||
|
if n.kind != notifyResolved && n.userID != nil && cfg.PublicURL != "" {
|
||||||
|
raw, err := issueAckToken(ctx, db, n.incidentID, *n.userID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("issue ack token: %w", err)
|
||||||
|
}
|
||||||
|
msg.Actions = append(msg.Actions, ntfyAction{
|
||||||
|
Action: "http",
|
||||||
|
Label: "Acknowledge",
|
||||||
|
URL: strings.TrimSuffix(cfg.PublicURL, "/") + "/api/notify/ack/" + raw,
|
||||||
|
Method: "POST",
|
||||||
|
Clear: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return publish(ctx, cfg, msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ntfyMessage is ntfy's JSON publish format. Using it rather than the X-Actions
|
||||||
|
// header avoids that header's comma and quote escaping rules, which are easy to
|
||||||
|
// break with a title that happens to contain a comma.
|
||||||
|
type ntfyMessage struct {
|
||||||
|
Topic string `json:"topic"`
|
||||||
|
Title string `json:"title,omitempty"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
Priority int `json:"priority,omitempty"`
|
||||||
|
Tags []string `json:"tags,omitempty"`
|
||||||
|
Click string `json:"click,omitempty"`
|
||||||
|
Actions []ntfyAction `json:"actions,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ntfyAction struct {
|
||||||
|
Action string `json:"action"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
Method string `json:"method,omitempty"`
|
||||||
|
Clear bool `json:"clear,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// renderNotification builds the message body for one queued notification.
|
||||||
|
func renderNotification(inc models.Incident, n outboxRow, firing int, cfg NotifyConfig) ntfyMessage {
|
||||||
|
msg := ntfyMessage{Topic: n.topic}
|
||||||
|
|
||||||
|
if cfg.PublicURL != "" {
|
||||||
|
// The web UI's page for the incident, so tapping the notification
|
||||||
|
// opens something a browser can use.
|
||||||
|
msg.Click = fmt.Sprintf("%s/incidents/%d",
|
||||||
|
strings.TrimSuffix(cfg.PublicURL, "/"), inc.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch n.kind {
|
||||||
|
case notifyResolved:
|
||||||
|
msg.Title = "Resolved: " + inc.Title
|
||||||
|
msg.Message = "All alerts stopped firing after " +
|
||||||
|
humanDuration(time.Since(inc.TriggeredAt))
|
||||||
|
msg.Priority = ntfyPriorityLow
|
||||||
|
msg.Tags = []string{"white_check_mark"}
|
||||||
|
return msg
|
||||||
|
|
||||||
|
case notifyReminder:
|
||||||
|
msg.Title = "Still unacknowledged: " + inc.Title
|
||||||
|
default:
|
||||||
|
msg.Title = inc.Title
|
||||||
|
}
|
||||||
|
|
||||||
|
severity := derefString(inc.Severity)
|
||||||
|
|
||||||
|
parts := []string{fmt.Sprintf("%d alert%s firing", firing, plural(firing))}
|
||||||
|
if severity != "" {
|
||||||
|
parts = append(parts, "severity "+severity)
|
||||||
|
}
|
||||||
|
if assignee := derefString(inc.AssignedToUser); assignee != "" {
|
||||||
|
parts = append(parts, "on call: "+assignee)
|
||||||
|
}
|
||||||
|
if n.kind == notifyReminder {
|
||||||
|
parts = append(parts, "open "+humanDuration(time.Since(inc.TriggeredAt)))
|
||||||
|
}
|
||||||
|
|
||||||
|
msg.Message = strings.Join(parts, " · ")
|
||||||
|
msg.Priority = ntfyPriority(severity)
|
||||||
|
msg.Tags = []string{severityTag(severity)}
|
||||||
|
return msg
|
||||||
|
}
|
||||||
|
|
||||||
|
// ntfy's priority scale. Max is the one that overrides the phone's quiet
|
||||||
|
// settings, which is the whole point of paging on critical.
|
||||||
|
const (
|
||||||
|
ntfyPriorityLow = 2
|
||||||
|
ntfyPriorityDefault = 3
|
||||||
|
ntfyPriorityHigh = 4
|
||||||
|
ntfyPriorityMax = 5
|
||||||
|
)
|
||||||
|
|
||||||
|
// ntfyPriority maps an incident's severity onto ntfy's scale, following the
|
||||||
|
// same ordering severityRank uses. An unrecognised severity gets the default
|
||||||
|
// rather than being silenced.
|
||||||
|
func ntfyPriority(severity string) int {
|
||||||
|
switch severityRank(severity) {
|
||||||
|
case 4:
|
||||||
|
return ntfyPriorityMax
|
||||||
|
case 3:
|
||||||
|
return ntfyPriorityHigh
|
||||||
|
case 2:
|
||||||
|
return ntfyPriorityDefault
|
||||||
|
case 1:
|
||||||
|
return ntfyPriorityLow
|
||||||
|
default:
|
||||||
|
return ntfyPriorityDefault
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func severityTag(severity string) string {
|
||||||
|
switch severityRank(severity) {
|
||||||
|
case 4:
|
||||||
|
return "rotating_light"
|
||||||
|
case 3:
|
||||||
|
return "red_circle"
|
||||||
|
case 2:
|
||||||
|
return "warning"
|
||||||
|
case 1:
|
||||||
|
return "information_source"
|
||||||
|
default:
|
||||||
|
return "bell"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// publish POSTs one message to ntfy.
|
||||||
|
func publish(ctx context.Context, cfg NotifyConfig, msg ntfyMessage) error {
|
||||||
|
body, err := json.Marshal(msg)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||||
|
strings.TrimSuffix(cfg.BaseURL, "/"), bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
if cfg.Token != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+cfg.Token)
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := notifyClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
return fmt.Errorf("ntfy returned %s", resp.Status)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// enqueueNotification adds one row to the outbox, due immediately.
|
||||||
|
func enqueueNotification(ctx context.Context, q querier, incidentID int64, userID *int64, topic, kind string) error {
|
||||||
|
now := time.Now().Unix()
|
||||||
|
_, err := q.ExecContext(ctx, `
|
||||||
|
INSERT INTO notifications (incident_id, user_id, topic, kind, created_at, send_after)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)`, incidentID, userID, topic, kind, now, now)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// notifyTarget decides where a newly opened incident's notification goes.
|
||||||
|
//
|
||||||
|
// The on-call user's own topic when they have one, otherwise the fallback
|
||||||
|
// topic with no user attached. Deliberately not "the fallback topic, attributed
|
||||||
|
// to the on-call user": the fallback is shared, and an Acknowledge button on a
|
||||||
|
// shared topic would let any subscriber acknowledge as somebody else.
|
||||||
|
func notifyTarget(ctx context.Context, q querier, cfg NotifyConfig, onCall *int64) (topic string, userID *int64) {
|
||||||
|
if onCall != nil {
|
||||||
|
var t *string
|
||||||
|
err := q.QueryRowContext(ctx,
|
||||||
|
"SELECT ntfy_topic FROM users WHERE id = ?", *onCall).Scan(&t)
|
||||||
|
if err == nil && t != nil && *t != "" {
|
||||||
|
return *t, onCall
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cfg.FallbackTopic, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// enqueueOpened queues the notification for a freshly opened incident. It is the
|
||||||
|
// only enqueue point that has to resolve a topic from scratch; every later
|
||||||
|
// notification for the incident reuses what this one chose.
|
||||||
|
func enqueueOpened(ctx context.Context, q querier, cfg NotifyConfig, incidentID int64, onCall *int64) error {
|
||||||
|
if !cfg.enabled() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
topic, userID := notifyTarget(ctx, q, cfg, onCall)
|
||||||
|
if topic == "" {
|
||||||
|
// Nobody on call has a topic and there is no fallback: there is nowhere
|
||||||
|
// to send this, and queueing it would only accumulate undeliverable rows.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return enqueueNotification(ctx, q, incidentID, userID, topic, notifyTriggered)
|
||||||
|
}
|
||||||
|
|
||||||
|
// enqueueResolved queues the all-clear, reusing the topic the incident's last
|
||||||
|
// notification went to. That needs no configuration to reach this function, and
|
||||||
|
// it gives the right rule for free: you only hear that something resolved if you
|
||||||
|
// were told it started.
|
||||||
|
func enqueueResolved(ctx context.Context, q querier, incidentID int64) error {
|
||||||
|
var topic string
|
||||||
|
var userID *int64
|
||||||
|
err := q.QueryRowContext(ctx, `
|
||||||
|
SELECT topic, user_id FROM notifications
|
||||||
|
WHERE incident_id = ? ORDER BY id DESC LIMIT 1`, incidentID).Scan(&topic, &userID)
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return enqueueNotification(ctx, q, incidentID, userID, topic, notifyResolved)
|
||||||
|
}
|
||||||
|
|
||||||
|
// humanDuration renders an age the way a person reads it at 3am: coarse, and
|
||||||
|
// never more than two units.
|
||||||
|
func humanDuration(d time.Duration) string {
|
||||||
|
if d < time.Minute {
|
||||||
|
return "less than a minute"
|
||||||
|
}
|
||||||
|
if d < time.Hour {
|
||||||
|
return fmt.Sprintf("%dm", int(d.Minutes()))
|
||||||
|
}
|
||||||
|
h := int(d.Hours())
|
||||||
|
m := int(d.Minutes()) - h*60
|
||||||
|
if m == 0 {
|
||||||
|
return fmt.Sprintf("%dh", h)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%dh%dm", h, m)
|
||||||
|
}
|
||||||
|
|
||||||
|
func plural(n int) string {
|
||||||
|
if n == 1 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return "s"
|
||||||
|
}
|
||||||
|
|
||||||
|
// derefString reads a nullable text column as a plain string.
|
||||||
|
func derefString(s *string) string {
|
||||||
|
if s == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return *s
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/hex"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// issueAckToken mints the secret behind one notification's Acknowledge button
|
||||||
|
// and returns the raw value to embed in its URL. Only the hash is stored, the
|
||||||
|
// same way api_keys works.
|
||||||
|
//
|
||||||
|
// A fresh token per delivery rather than one per incident: the raw value only
|
||||||
|
// exists for as long as it takes to build the message, so there is nothing to
|
||||||
|
// look up and reuse later, and a reminder that supersedes an earlier page
|
||||||
|
// carries its own credential.
|
||||||
|
func issueAckToken(ctx context.Context, q querier, incidentID, userID int64) (string, error) {
|
||||||
|
raw, hash, err := randomToken()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
if _, err := q.ExecContext(ctx, `
|
||||||
|
INSERT INTO incident_ack_tokens (token_hash, incident_id, user_id, created_at, expires_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?)`,
|
||||||
|
hash, incidentID, userID, now.Unix(), now.Add(ackTokenTTL).Unix()); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return raw, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleNotifyAck acknowledges an incident from the Acknowledge button in a
|
||||||
|
// push notification.
|
||||||
|
//
|
||||||
|
// It is deliberately outside AuthMiddleware: the caller is a phone acting on a
|
||||||
|
// notification, not a client holding an API key. What stands in for the key is
|
||||||
|
// the token in the path — 256 bits of entropy, valid for one incident, one
|
||||||
|
// action, and one day. It must stay publicly reachable for the button to work
|
||||||
|
// when the responder is off the cluster network.
|
||||||
|
func handleNotifyAck(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
h := sha256.Sum256([]byte(chi.URLParam(r, "token")))
|
||||||
|
hash := hex.EncodeToString(h[:])
|
||||||
|
|
||||||
|
var incidentID, userID int64
|
||||||
|
err := db.QueryRowContext(r.Context(), `
|
||||||
|
SELECT incident_id, user_id FROM incident_ack_tokens
|
||||||
|
WHERE token_hash = ? AND expires_at > ?`,
|
||||||
|
hash, time.Now().Unix()).Scan(&incidentID, &userID)
|
||||||
|
if err != nil {
|
||||||
|
// Unknown and expired get the same answer, so the endpoint cannot be
|
||||||
|
// used to probe which tokens once existed.
|
||||||
|
respond(w, http.StatusNotFound, errResp("invalid or expired token"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
acked, err := acknowledgeIncident(r.Context(), db, incidentID, userID)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !acked {
|
||||||
|
// The incident closed between the page and the tap. Nothing to do,
|
||||||
|
// and nothing the responder did wrong — report the state, not an error,
|
||||||
|
// so ntfy shows a success toast rather than a failure.
|
||||||
|
respond(w, http.StatusOK, map[string]any{
|
||||||
|
"incident_id": incidentID,
|
||||||
|
"status": "resolved",
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respond(w, http.StatusOK, map[string]any{
|
||||||
|
"incident_id": incidentID,
|
||||||
|
"status": "acknowledged",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// purgeAckTokens drops tokens whose notifications are long past. Nothing else
|
||||||
|
// deletes them: incidents are archived rather than removed, so the cascade never
|
||||||
|
// fires in practice.
|
||||||
|
func purgeAckTokens(ctx context.Context, db *sql.DB) {
|
||||||
|
res, err := db.ExecContext(ctx,
|
||||||
|
"DELETE FROM incident_ack_tokens WHERE expires_at < ?", time.Now().Unix())
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("sweeper: purge ack tokens: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n > 0 {
|
||||||
|
log.Printf("sweeper: purged %d expired ack token(s)", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,724 @@
|
|||||||
|
package api_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/api"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Fake ntfy
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// pushed is one message the fake ntfy received, in ntfy's JSON publish shape.
|
||||||
|
type pushed struct {
|
||||||
|
Topic string `json:"topic"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
Priority int `json:"priority"`
|
||||||
|
Tags []string `json:"tags"`
|
||||||
|
Click string `json:"click"`
|
||||||
|
Actions []struct {
|
||||||
|
Action string `json:"action"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
Method string `json:"method"`
|
||||||
|
Clear bool `json:"clear"`
|
||||||
|
} `json:"actions"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeNtfy records what the notifier published. status controls the reply, so a
|
||||||
|
// test can make delivery fail.
|
||||||
|
type fakeNtfy struct {
|
||||||
|
*httptest.Server
|
||||||
|
mu sync.Mutex
|
||||||
|
got []pushed
|
||||||
|
status int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFakeNtfy(t *testing.T) *fakeNtfy {
|
||||||
|
t.Helper()
|
||||||
|
f := &fakeNtfy{status: http.StatusOK}
|
||||||
|
f.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var msg pushed
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&msg); err != nil {
|
||||||
|
http.Error(w, "bad json", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
f.got = append(f.got, msg)
|
||||||
|
status := f.status
|
||||||
|
f.mu.Unlock()
|
||||||
|
w.WriteHeader(status)
|
||||||
|
}))
|
||||||
|
t.Cleanup(f.Close)
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeNtfy) messages() []pushed {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return append([]pushed(nil), f.got...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeNtfy) failWith(status int) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.status = status
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Harness
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// notifyTS builds a server with notifications enabled, the admin on call today,
|
||||||
|
// and a topic on the admin — the setup every delivery test needs.
|
||||||
|
func notifyTS(t *testing.T, cfg api.NotifyConfig) (*ts, *fakeNtfy) {
|
||||||
|
t.Helper()
|
||||||
|
f := newFakeNtfy(t)
|
||||||
|
cfg.BaseURL = f.URL
|
||||||
|
s := newTS(t, cfg)
|
||||||
|
|
||||||
|
putOnCall(t, s, 1)
|
||||||
|
setTopic(t, s, 1, "terdut-admin")
|
||||||
|
return s, f
|
||||||
|
}
|
||||||
|
|
||||||
|
func putOnCall(t *testing.T, s *ts, userID int) {
|
||||||
|
t.Helper()
|
||||||
|
today := time.Now().UTC().Format("2006-01-02")
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/schedule",
|
||||||
|
map[string]any{"user_id": userID, "dates": []string{today}})
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusCreated {
|
||||||
|
t.Fatalf("schedule assignment returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setTopic(t *testing.T, s *ts, userID int, topic string) {
|
||||||
|
t.Helper()
|
||||||
|
resp := s.req(t, http.MethodPut,
|
||||||
|
fmt.Sprintf("/api/users/%d/notify", userID), map[string]any{"ntfy_topic": topic})
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("set notify topic returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *ts) sweepNotify(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
api.NotifySweep(context.Background(), s.db, s.notify)
|
||||||
|
}
|
||||||
|
|
||||||
|
// countNotifications reports how many outbox rows exist, optionally of one kind.
|
||||||
|
func (s *ts) countNotifications(t *testing.T, kind string) int {
|
||||||
|
t.Helper()
|
||||||
|
var n int
|
||||||
|
query := "SELECT COUNT(*) FROM notifications"
|
||||||
|
args := []any{}
|
||||||
|
if kind != "" {
|
||||||
|
query += " WHERE kind = ?"
|
||||||
|
args = append(args, kind)
|
||||||
|
}
|
||||||
|
if err := s.db.QueryRow(query, args...).Scan(&n); err != nil {
|
||||||
|
t.Fatalf("count notifications: %v", err)
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// fireCritical posts a single critical alert, which opens one incident.
|
||||||
|
func fireCritical(t *testing.T, s *ts) {
|
||||||
|
t.Helper()
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-notify", "DiskFull", "firing", "2026-05-20T10:00:00Z", zeroTime,
|
||||||
|
map[string]string{"severity": "critical"}),
|
||||||
|
}, "{}:{alertname=\"DiskFull\"}")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Delivery
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestNotify_TriggeredIncidentPagesOnCall(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
|
||||||
|
if got := s.countNotifications(t, "triggered"); got != 1 {
|
||||||
|
t.Fatalf("expected 1 queued notification, got %d", got)
|
||||||
|
}
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
msgs := f.messages()
|
||||||
|
if len(msgs) != 1 {
|
||||||
|
t.Fatalf("expected 1 push, got %d", len(msgs))
|
||||||
|
}
|
||||||
|
m := msgs[0]
|
||||||
|
|
||||||
|
if m.Topic != "terdut-admin" {
|
||||||
|
t.Errorf("expected the on-call user's topic, got %q", m.Topic)
|
||||||
|
}
|
||||||
|
if m.Priority != 5 {
|
||||||
|
t.Errorf("expected max priority for a critical incident, got %d", m.Priority)
|
||||||
|
}
|
||||||
|
if !strings.Contains(m.Title, "DiskFull") {
|
||||||
|
t.Errorf("expected the incident title in %q", m.Title)
|
||||||
|
}
|
||||||
|
if !strings.Contains(m.Message, "severity critical") {
|
||||||
|
t.Errorf("expected the severity in %q", m.Message)
|
||||||
|
}
|
||||||
|
if m.Click != "https://terdut.example.com/incidents/1" {
|
||||||
|
t.Errorf("unexpected click target %q", m.Click)
|
||||||
|
}
|
||||||
|
if len(m.Actions) != 1 || m.Actions[0].Label != "Acknowledge" {
|
||||||
|
t.Fatalf("expected an Acknowledge action, got %+v", m.Actions)
|
||||||
|
}
|
||||||
|
if m.Actions[0].Method != http.MethodPost {
|
||||||
|
t.Errorf("expected the action to POST, got %q", m.Actions[0].Method)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A delivered row must not be delivered again on the next pass.
|
||||||
|
func TestNotify_DeliveredOnlyOnce(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
if got := len(f.messages()); got != 1 {
|
||||||
|
t.Errorf("expected 1 push across two passes, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Delivery on the timeline
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// notifyEvents picks the notifier's entries out of an incident's timeline.
|
||||||
|
// Asserted through the API rather than the table: the timeline is what the
|
||||||
|
// clients read, so its shape is the contract worth covering.
|
||||||
|
func notifyEvents(t *testing.T, s *ts, id int) []map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
var out []map[string]any
|
||||||
|
for _, e := range timeline(t, s, id) {
|
||||||
|
if e["type"] == "notified" || e["type"] == "notify_failed" {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotify_DeliveryIsRecordedOnTheTimeline(t *testing.T) {
|
||||||
|
s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
// Queued is not notified: nothing is on the timeline until ntfy accepts it.
|
||||||
|
if got := notifyEvents(t, s, 1); len(got) != 0 {
|
||||||
|
t.Fatalf("expected no event before delivery, got %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
events := notifyEvents(t, s, 1)
|
||||||
|
if len(events) != 1 {
|
||||||
|
t.Fatalf("expected 1 notification event, got %v", events)
|
||||||
|
}
|
||||||
|
e := events[0]
|
||||||
|
if e["type"] != "notified" {
|
||||||
|
t.Errorf("expected a notified event, got %v", e["type"])
|
||||||
|
}
|
||||||
|
if e["detail"] != "triggered" {
|
||||||
|
t.Errorf("expected the kind in detail, got %v", e["detail"])
|
||||||
|
}
|
||||||
|
if e["username"] != "admin" {
|
||||||
|
t.Errorf("expected the paged user attached, got %v", e["username"])
|
||||||
|
}
|
||||||
|
// The topic is a shared secret with ntfy; the timeline is not the place for it.
|
||||||
|
for _, v := range e {
|
||||||
|
if s, ok := v.(string); ok && strings.Contains(s, "terdut-admin") {
|
||||||
|
t.Errorf("expected the topic kept out of the timeline, found it in %v", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A redelivery-free pass must not double-log either.
|
||||||
|
func TestNotify_TimelineRecordsOneEventPerDelivery(t *testing.T) {
|
||||||
|
s, _ := notifyTS(t, api.NotifyConfig{
|
||||||
|
PublicURL: "https://terdut.example.com",
|
||||||
|
RepeatEvery: 15 * time.Minute,
|
||||||
|
})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
s.ageNotifications(t, 20*time.Minute)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
events := notifyEvents(t, s, 1)
|
||||||
|
if len(events) != 2 {
|
||||||
|
t.Fatalf("expected one event per delivery, got %v", events)
|
||||||
|
}
|
||||||
|
if events[0]["detail"] != "triggered" || events[1]["detail"] != "reminder" {
|
||||||
|
t.Errorf("expected triggered then reminder, got %v and %v",
|
||||||
|
events[0]["detail"], events[1]["detail"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotify_AllClearIsRecordedOnTheTimeline(t *testing.T) {
|
||||||
|
s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-notify", "DiskFull", "resolved", "2026-05-20T10:00:00Z",
|
||||||
|
"2026-05-20T11:00:00Z", map[string]string{"severity": "critical"}),
|
||||||
|
}, "{}:{alertname=\"DiskFull\"}")
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
events := notifyEvents(t, s, 1)
|
||||||
|
if len(events) != 2 {
|
||||||
|
t.Fatalf("expected the all-clear recorded, got %v", events)
|
||||||
|
}
|
||||||
|
if events[1]["detail"] != "resolved" {
|
||||||
|
t.Errorf("expected a resolved event, got %v", events[1]["detail"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A page to the shared fallback belongs to nobody, and the timeline has to say
|
||||||
|
// so rather than attributing it to whoever happens to be on call now.
|
||||||
|
func TestNotify_FallbackDeliveryHasNoUser(t *testing.T) {
|
||||||
|
f := newFakeNtfy(t)
|
||||||
|
s := newTS(t, api.NotifyConfig{
|
||||||
|
BaseURL: f.URL,
|
||||||
|
FallbackTopic: "terdut-oncall",
|
||||||
|
PublicURL: "https://terdut.example.com",
|
||||||
|
})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
events := notifyEvents(t, s, 1)
|
||||||
|
if len(events) != 1 {
|
||||||
|
t.Fatalf("expected 1 notification event, got %v", events)
|
||||||
|
}
|
||||||
|
if got, ok := events[0]["username"]; ok && got != nil && got != "" {
|
||||||
|
t.Errorf("expected no user on a fallback-topic page, got %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The failure worth seeing: nobody was paged, and the timeline says so instead
|
||||||
|
// of looking exactly like a delivery that worked.
|
||||||
|
func TestNotify_ExhaustedRetriesAreRecordedOnce(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
f.failWith(http.StatusInternalServerError)
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
// One pass per attempt, each made due by clearing the backoff the last one set.
|
||||||
|
for i := 0; i < 10; i++ {
|
||||||
|
s.sweepNotify(t)
|
||||||
|
s.exec(t, "UPDATE notifications SET send_after = ? WHERE sent_at IS NULL",
|
||||||
|
time.Now().Add(-time.Second).Unix())
|
||||||
|
}
|
||||||
|
|
||||||
|
events := notifyEvents(t, s, 1)
|
||||||
|
if len(events) != 1 {
|
||||||
|
t.Fatalf("expected exactly one failure event, got %v", events)
|
||||||
|
}
|
||||||
|
if events[0]["type"] != "notify_failed" {
|
||||||
|
t.Errorf("expected notify_failed, got %v", events[0]["type"])
|
||||||
|
}
|
||||||
|
detail, _ := events[0]["detail"].(string)
|
||||||
|
if !strings.HasPrefix(detail, "triggered: ") || !strings.Contains(detail, "500") {
|
||||||
|
t.Errorf("expected the kind and the reason in %q", detail)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Acknowledging from the notification
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestNotify_AckButtonAcknowledgesIncident(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
ackURL := f.messages()[0].Actions[0].URL
|
||||||
|
// The action URL is built for the public hostname; point it at the test
|
||||||
|
// server, which is the same handler.
|
||||||
|
path := ackURL[strings.Index(ackURL, "/api/notify/ack/"):]
|
||||||
|
|
||||||
|
resp, err := http.Post(s.URL+path, "application/json", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ack: %v", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200 from the ack button, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
inc := getIncident(t, s, 1)
|
||||||
|
if inc["status"] != "acknowledged" {
|
||||||
|
t.Errorf("expected the incident acknowledged, got %v", inc["status"])
|
||||||
|
}
|
||||||
|
if inc["acknowledged_by"] != "admin" {
|
||||||
|
t.Errorf("expected the ack attributed to the token's user, got %v", inc["acknowledged_by"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// The timeline must record it the same way the authenticated route would.
|
||||||
|
events := timeline(t, s, 1)
|
||||||
|
if !contains(eventTypes(events), "acknowledged") {
|
||||||
|
t.Errorf("expected an acknowledged event, got %v", eventTypes(events))
|
||||||
|
}
|
||||||
|
for _, e := range events {
|
||||||
|
if e["type"] == "acknowledged" && e["username"] != "admin" {
|
||||||
|
t.Errorf("expected the acknowledged event attributed to admin, got %v", e["username"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotify_AckRejectsUnknownToken(t *testing.T) {
|
||||||
|
s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
fireCritical(t, s)
|
||||||
|
|
||||||
|
resp, err := http.Post(s.URL+"/api/notify/ack/deadbeef", "application/json", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ack: %v", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusNotFound {
|
||||||
|
t.Errorf("expected 404 for an unknown token, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
|
||||||
|
t.Errorf("expected the incident untouched, got %v", inc["status"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotify_AckRejectsExpiredToken(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
ackURL := f.messages()[0].Actions[0].URL
|
||||||
|
path := ackURL[strings.Index(ackURL, "/api/notify/ack/"):]
|
||||||
|
|
||||||
|
// Age the token past its TTL. The token's inputs are wall-clock timestamps,
|
||||||
|
// so this is the same trick the sweeper tests use.
|
||||||
|
s.exec(t, "UPDATE incident_ack_tokens SET expires_at = ?", time.Now().Add(-time.Minute).Unix())
|
||||||
|
|
||||||
|
resp, err := http.Post(s.URL+path, "application/json", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ack: %v", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusNotFound {
|
||||||
|
t.Errorf("expected 404 for an expired token, got %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
if inc := getIncident(t, s, 1); inc["status"] != "triggered" {
|
||||||
|
t.Errorf("expected the incident untouched, got %v", inc["status"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The sweeper is what stops expired tokens accumulating forever.
|
||||||
|
func TestNotify_SweepPurgesExpiredAckTokens(t *testing.T) {
|
||||||
|
s, _ := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
s.exec(t, "UPDATE incident_ack_tokens SET expires_at = ?", time.Now().Add(-time.Minute).Unix())
|
||||||
|
|
||||||
|
api.Sweep(context.Background(), s.db, 168*time.Hour, 6*time.Hour, s.deadman, s.notify)
|
||||||
|
|
||||||
|
var n int
|
||||||
|
if err := s.db.QueryRow("SELECT COUNT(*) FROM incident_ack_tokens").Scan(&n); err != nil {
|
||||||
|
t.Fatalf("count tokens: %v", err)
|
||||||
|
}
|
||||||
|
if n != 0 {
|
||||||
|
t.Errorf("expected expired tokens purged, %d left", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Reminders
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// ageNotifications backdates every sent notification so the next pass sees the
|
||||||
|
// reminder as due.
|
||||||
|
func (s *ts) ageNotifications(t *testing.T, by time.Duration) {
|
||||||
|
t.Helper()
|
||||||
|
s.exec(t, "UPDATE notifications SET created_at = ? WHERE sent_at IS NOT NULL",
|
||||||
|
time.Now().Add(-by).Unix())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotify_UnacknowledgedIncidentIsRenotified(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{
|
||||||
|
PublicURL: "https://terdut.example.com",
|
||||||
|
RepeatEvery: 15 * time.Minute,
|
||||||
|
})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
s.ageNotifications(t, 20*time.Minute)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
msgs := f.messages()
|
||||||
|
if len(msgs) != 2 {
|
||||||
|
t.Fatalf("expected a reminder push, got %d message(s)", len(msgs))
|
||||||
|
}
|
||||||
|
if !strings.Contains(msgs[1].Title, "Still unacknowledged") {
|
||||||
|
t.Errorf("expected the reminder to say so, got %q", msgs[1].Title)
|
||||||
|
}
|
||||||
|
if msgs[1].Topic != "terdut-admin" {
|
||||||
|
t.Errorf("expected the reminder on the same topic, got %q", msgs[1].Topic)
|
||||||
|
}
|
||||||
|
// Each page carries its own credential.
|
||||||
|
if len(msgs[1].Actions) != 1 || msgs[1].Actions[0].URL == msgs[0].Actions[0].URL {
|
||||||
|
t.Errorf("expected the reminder to carry a fresh ack token")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotify_AcknowledgedIncidentStopsReminders(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{
|
||||||
|
PublicURL: "https://terdut.example.com",
|
||||||
|
RepeatEvery: 15 * time.Minute,
|
||||||
|
})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/acknowledge", nil)
|
||||||
|
resp.Body.Close()
|
||||||
|
|
||||||
|
s.ageNotifications(t, 20*time.Minute)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
if got := len(f.messages()); got != 1 {
|
||||||
|
t.Errorf("expected no reminder once acknowledged, got %d message(s)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snooze is the deliberate "not now", and it is what mutes the pager.
|
||||||
|
func TestNotify_SnoozedIncidentStopsReminders(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{
|
||||||
|
PublicURL: "https://terdut.example.com",
|
||||||
|
RepeatEvery: 15 * time.Minute,
|
||||||
|
})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/snooze", map[string]any{"duration": "1h"})
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
t.Fatalf("snooze returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
|
||||||
|
s.ageNotifications(t, 20*time.Minute)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
if got := len(f.messages()); got != 1 {
|
||||||
|
t.Errorf("expected no reminder while snoozed, got %d message(s)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNotify_ZeroRepeatDisablesReminders(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
s.ageNotifications(t, 24*time.Hour)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
if got := len(f.messages()); got != 1 {
|
||||||
|
t.Errorf("expected reminders off, got %d message(s)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Resolution
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestNotify_ResolvedIncidentSendsAllClear(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
postWebhook(t, s, []map[string]any{
|
||||||
|
amAlert("fp-notify", "DiskFull", "resolved", "2026-05-20T10:00:00Z",
|
||||||
|
"2026-05-20T11:00:00Z", map[string]string{"severity": "critical"}),
|
||||||
|
}, "{}:{alertname=\"DiskFull\"}")
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
msgs := f.messages()
|
||||||
|
if len(msgs) != 2 {
|
||||||
|
t.Fatalf("expected an all-clear push, got %d message(s)", len(msgs))
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(msgs[1].Title, "Resolved:") {
|
||||||
|
t.Errorf("expected a resolved title, got %q", msgs[1].Title)
|
||||||
|
}
|
||||||
|
if msgs[1].Priority != 2 {
|
||||||
|
t.Errorf("expected the all-clear at low priority, got %d", msgs[1].Priority)
|
||||||
|
}
|
||||||
|
// Nothing to acknowledge on a closed incident.
|
||||||
|
if len(msgs[1].Actions) != 0 {
|
||||||
|
t.Errorf("expected no actions on the all-clear, got %+v", msgs[1].Actions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Closing an incident by hand sends nothing: the person who did it knows.
|
||||||
|
func TestNotify_ManualResolveSendsNothing(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
resp := s.req(t, http.MethodPost, "/api/incidents/1/resolve", nil)
|
||||||
|
resp.Body.Close()
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
if got := len(f.messages()); got != 1 {
|
||||||
|
t.Errorf("expected no push for a manual resolve, got %d message(s)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Routing and configuration
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// With nobody on call the page goes to the shared fallback, and carries no
|
||||||
|
// Acknowledge button — there is no user to attribute the acknowledgement to.
|
||||||
|
func TestNotify_FallbackTopicHasNoAckButton(t *testing.T) {
|
||||||
|
f := newFakeNtfy(t)
|
||||||
|
s := newTS(t, api.NotifyConfig{
|
||||||
|
BaseURL: f.URL,
|
||||||
|
FallbackTopic: "terdut-oncall",
|
||||||
|
PublicURL: "https://terdut.example.com",
|
||||||
|
})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
msgs := f.messages()
|
||||||
|
if len(msgs) != 1 {
|
||||||
|
t.Fatalf("expected 1 push, got %d", len(msgs))
|
||||||
|
}
|
||||||
|
if msgs[0].Topic != "terdut-oncall" {
|
||||||
|
t.Errorf("expected the fallback topic, got %q", msgs[0].Topic)
|
||||||
|
}
|
||||||
|
if len(msgs[0].Actions) != 0 {
|
||||||
|
t.Errorf("expected no ack button on a shared topic, got %+v", msgs[0].Actions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nobody on call and no fallback means there is nowhere to send: queueing would
|
||||||
|
// only pile up rows that can never be delivered.
|
||||||
|
func TestNotify_NoTargetQueuesNothing(t *testing.T) {
|
||||||
|
f := newFakeNtfy(t)
|
||||||
|
s := newTS(t, api.NotifyConfig{BaseURL: f.URL})
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
|
||||||
|
if got := s.countNotifications(t, ""); got != 0 {
|
||||||
|
t.Errorf("expected nothing queued without a target, got %d", got)
|
||||||
|
}
|
||||||
|
s.sweepNotify(t)
|
||||||
|
if got := len(f.messages()); got != 0 {
|
||||||
|
t.Errorf("expected no push, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The zero NotifyConfig is what every pre-existing test runs under.
|
||||||
|
func TestNotify_DisabledQueuesNothing(t *testing.T) {
|
||||||
|
s := newTS(t)
|
||||||
|
putOnCall(t, s, 1)
|
||||||
|
setTopic(t, s, 1, "terdut-admin")
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
|
||||||
|
if got := s.countNotifications(t, ""); got != 0 {
|
||||||
|
t.Errorf("expected nothing queued with notifications off, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Retries
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func TestNotify_FailedDeliveryRetriesWithBackoff(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{PublicURL: "https://terdut.example.com"})
|
||||||
|
f.failWith(http.StatusInternalServerError)
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
var attempts int
|
||||||
|
var sentAt *int64
|
||||||
|
var sendAfter int64
|
||||||
|
var lastError *string
|
||||||
|
if err := s.db.QueryRow(
|
||||||
|
"SELECT attempts, sent_at, send_after, last_error FROM notifications WHERE id = 1").
|
||||||
|
Scan(&attempts, &sentAt, &sendAfter, &lastError); err != nil {
|
||||||
|
t.Fatalf("read notification: %v", err)
|
||||||
|
}
|
||||||
|
if attempts != 1 {
|
||||||
|
t.Errorf("expected 1 attempt recorded, got %d", attempts)
|
||||||
|
}
|
||||||
|
if sentAt != nil {
|
||||||
|
t.Errorf("expected the row unsent, got sent_at %v", *sentAt)
|
||||||
|
}
|
||||||
|
if sendAfter <= time.Now().Unix() {
|
||||||
|
t.Errorf("expected the retry pushed into the future, got %d", sendAfter)
|
||||||
|
}
|
||||||
|
if lastError == nil || !strings.Contains(*lastError, "500") {
|
||||||
|
t.Errorf("expected the failure recorded, got %v", lastError)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Backing off means the next pass leaves it alone until it is due.
|
||||||
|
s.sweepNotify(t)
|
||||||
|
if got := len(f.messages()); got != 1 {
|
||||||
|
t.Errorf("expected no immediate retry, got %d attempt(s)", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once due and once ntfy recovers, it goes out.
|
||||||
|
f.failWith(http.StatusOK)
|
||||||
|
s.exec(t, "UPDATE notifications SET send_after = ? WHERE id = 1", time.Now().Add(-time.Second).Unix())
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
if err := s.db.QueryRow("SELECT sent_at FROM notifications WHERE id = 1").Scan(&sentAt); err != nil {
|
||||||
|
t.Fatalf("read notification: %v", err)
|
||||||
|
}
|
||||||
|
if sentAt == nil {
|
||||||
|
t.Error("expected the retry to succeed once ntfy recovered")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ntfy outage must not produce a reminder backlog that all lands at once
|
||||||
|
// when it comes back: the previous page has to have been sent first.
|
||||||
|
func TestNotify_UnsentNotificationBlocksReminders(t *testing.T) {
|
||||||
|
s, f := notifyTS(t, api.NotifyConfig{
|
||||||
|
PublicURL: "https://terdut.example.com",
|
||||||
|
RepeatEvery: 15 * time.Minute,
|
||||||
|
})
|
||||||
|
f.failWith(http.StatusInternalServerError)
|
||||||
|
|
||||||
|
fireCritical(t, s)
|
||||||
|
s.sweepNotify(t)
|
||||||
|
s.exec(t, "UPDATE notifications SET created_at = ?", time.Now().Add(-time.Hour).Unix())
|
||||||
|
s.sweepNotify(t)
|
||||||
|
|
||||||
|
if got := s.countNotifications(t, "reminder"); got != 0 {
|
||||||
|
t.Errorf("expected no reminders queued behind an undelivered page, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
+51
-10
@@ -4,11 +4,16 @@ import (
|
|||||||
"database/sql"
|
"database/sql"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/web"
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/go-chi/chi/v5/middleware"
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
func NewRouter(db *sql.DB) http.Handler {
|
// NewRouter builds the HTTP surface. notify and deadman are passed through to
|
||||||
|
// the webhook, the only handler that has to decide where a new incident's page
|
||||||
|
// goes and which arriving alerts are heartbeats rather than problems. A zero
|
||||||
|
// notify disables notifications; a zero deadman disables dead man's switches.
|
||||||
|
func NewRouter(db *sql.DB, notify NotifyConfig, deadman DeadmanConfig) http.Handler {
|
||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
r.Use(middleware.Logger)
|
r.Use(middleware.Logger)
|
||||||
r.Use(middleware.Recoverer)
|
r.Use(middleware.Recoverer)
|
||||||
@@ -17,40 +22,76 @@ func NewRouter(db *sql.DB) http.Handler {
|
|||||||
respond(w, http.StatusOK, map[string]string{"status": "ok"})
|
respond(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||||
})
|
})
|
||||||
|
|
||||||
// Unauthenticated: bootstrap and Alertmanager webhook receiver.
|
// Unauthenticated: bootstrap, the Alertmanager webhook receiver, and the
|
||||||
|
// Acknowledge button in a push notification. The last one is authorised by
|
||||||
|
// the scoped token in its path rather than an API key, and has to stay
|
||||||
|
// reachable from outside the cluster for the button to work.
|
||||||
r.Post("/api/bootstrap", handleBootstrap(db))
|
r.Post("/api/bootstrap", handleBootstrap(db))
|
||||||
r.Post("/api/alertmanager/webhook", handleAlertmanagerWebhook(db))
|
r.Post("/api/alertmanager/webhook", handleAlertmanagerWebhook(db, notify, deadman))
|
||||||
|
r.Post("/api/notify/ack/{token}", handleNotifyAck(db))
|
||||||
|
|
||||||
|
// Signing in to the web UI. Login trades a password for a session cookie,
|
||||||
|
// which AuthMiddleware accepts in place of an API key.
|
||||||
|
r.Post("/api/login", handleLogin(db, newLoginLimiter(), notify.PublicURL))
|
||||||
|
r.Post("/api/logout", handleLogout(db, notify.PublicURL))
|
||||||
|
|
||||||
// All other /api routes require a valid API key.
|
// All other /api routes require a valid API key.
|
||||||
r.Group(func(r chi.Router) {
|
r.Group(func(r chi.Router) {
|
||||||
r.Use(AuthMiddleware(db))
|
r.Use(AuthMiddleware(db))
|
||||||
|
|
||||||
|
r.Get("/api/me", handleMe(db))
|
||||||
r.Get("/api/users", handleListUsers(db))
|
r.Get("/api/users", handleListUsers(db))
|
||||||
r.Post("/api/users", handleCreateUser(db))
|
r.Post("/api/users", handleCreateUser(db))
|
||||||
r.Delete("/api/users/{id}", handleDeleteUser(db))
|
r.Delete("/api/users/{id}", handleDeleteUser(db))
|
||||||
|
r.Put("/api/users/{id}/notify", handleSetNotifyTarget(db))
|
||||||
|
r.Put("/api/users/{id}/password", handleSetPassword(db))
|
||||||
r.Post("/api/users/{id}/api-keys", handleCreateAPIKey(db))
|
r.Post("/api/users/{id}/api-keys", handleCreateAPIKey(db))
|
||||||
r.Delete("/api/users/{id}/api-keys/{keyID}", handleDeleteAPIKey(db))
|
r.Delete("/api/users/{id}/api-keys/{keyID}", handleDeleteAPIKey(db))
|
||||||
|
|
||||||
|
// Alerts are read-only: they are Alertmanager's record, not a work
|
||||||
|
// queue. Everything a person does happens on the incident instead.
|
||||||
r.Get("/api/alerts", handleListAlerts(db))
|
r.Get("/api/alerts", handleListAlerts(db))
|
||||||
r.Get("/api/alerts/{id}", handleGetAlert(db))
|
r.Get("/api/alerts/{id}", handleGetAlert(db))
|
||||||
r.Post("/api/alerts/{id}/acknowledge", handleAcknowledge(db))
|
|
||||||
r.Delete("/api/alerts/{id}/acknowledge", handleUnacknowledge(db))
|
r.Get("/api/incidents", handleListIncidents(db))
|
||||||
r.Post("/api/alerts/{id}/archive", handleArchive(db))
|
r.Get("/api/incidents/{id}", handleGetIncident(db))
|
||||||
r.Delete("/api/alerts/{id}/archive", handleUnarchive(db))
|
r.Get("/api/incidents/{id}/alerts", handleIncidentAlerts(db))
|
||||||
r.Get("/api/alerts/{id}/comments", handleListComments(db))
|
r.Get("/api/incidents/{id}/timeline", handleIncidentTimeline(db))
|
||||||
r.Post("/api/alerts/{id}/comments", handleCreateComment(db))
|
r.Post("/api/incidents/{id}/acknowledge", handleIncidentAcknowledge(db))
|
||||||
r.Delete("/api/alerts/{id}/comments/{commentID}", handleDeleteComment(db))
|
r.Delete("/api/incidents/{id}/acknowledge", handleIncidentUnacknowledge(db))
|
||||||
|
r.Post("/api/incidents/{id}/resolve", handleIncidentResolve(db))
|
||||||
|
r.Post("/api/incidents/{id}/assign", handleIncidentAssign(db))
|
||||||
|
r.Post("/api/incidents/{id}/snooze", handleIncidentSnooze(db))
|
||||||
|
r.Delete("/api/incidents/{id}/snooze", handleIncidentUnsnooze(db))
|
||||||
|
r.Post("/api/incidents/{id}/archive", handleIncidentArchive(db))
|
||||||
|
r.Delete("/api/incidents/{id}/archive", handleIncidentUnarchive(db))
|
||||||
|
r.Post("/api/incidents/{id}/notes", handleCreateNote(db))
|
||||||
|
r.Delete("/api/incidents/{id}/notes/{eventID}", handleDeleteNote(db))
|
||||||
|
|
||||||
r.Post("/api/schedule", handleCreateSchedule(db))
|
r.Post("/api/schedule", handleCreateSchedule(db))
|
||||||
r.Get("/api/schedule/current", handleCurrentSchedule(db)) // must be before /{id}
|
r.Get("/api/schedule/current", handleCurrentSchedule(db)) // must be before /{id}
|
||||||
r.Get("/api/schedule", handleListSchedule(db))
|
r.Get("/api/schedule", handleListSchedule(db))
|
||||||
r.Delete("/api/schedule/{id}", handleDeleteSchedule(db))
|
r.Delete("/api/schedule/{id}", handleDeleteSchedule(db))
|
||||||
|
|
||||||
|
r.Get("/api/stats/incidents", handleStatsIncidents(db))
|
||||||
r.Get("/api/stats/alerts", handleStatsAlerts(db))
|
r.Get("/api/stats/alerts", handleStatsAlerts(db))
|
||||||
r.Get("/api/stats/alerts/top", handleStatsTop(db))
|
r.Get("/api/stats/alerts/top", handleStatsTop(db))
|
||||||
r.Get("/api/stats/alerts/by-hour", handleStatsByHour(db))
|
r.Get("/api/stats/alerts/by-hour", handleStatsByHour(db))
|
||||||
r.Get("/api/stats/alerts/by-day", handleStatsByDay(db))
|
r.Get("/api/stats/alerts/by-day", handleStatsByDay(db))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Anything else under /api is a mistake in a client, and should say so in
|
||||||
|
// JSON rather than get the web UI's HTML.
|
||||||
|
r.Handle("/api/*", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
respond(w, http.StatusNotFound, errResp("not found"))
|
||||||
|
}))
|
||||||
|
|
||||||
|
// Everything outside /api is the web UI.
|
||||||
|
site, err := web.Handler()
|
||||||
|
if err != nil {
|
||||||
|
panic(err) // the site is embedded at build time; this cannot fail at runtime
|
||||||
|
}
|
||||||
|
r.Handle("/*", site)
|
||||||
|
|
||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/models"
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/yeniklas/terdut-server/internal/models"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func handleCreateSchedule(db *sql.DB) http.HandlerFunc {
|
func handleCreateSchedule(db *sql.DB) http.HandlerFunc {
|
||||||
@@ -17,6 +17,12 @@ func handleCreateSchedule(db *sql.DB) http.HandlerFunc {
|
|||||||
var req struct {
|
var req struct {
|
||||||
UserID int64 `json:"user_id"`
|
UserID int64 `json:"user_id"`
|
||||||
Dates []string `json:"dates"`
|
Dates []string `json:"dates"`
|
||||||
|
|
||||||
|
// Replace takes dates that somebody else already holds. It defaults
|
||||||
|
// to off so that the plain call cannot quietly move a shift off the
|
||||||
|
// person expecting to be paged for it — reassigning has to be asked
|
||||||
|
// for.
|
||||||
|
Replace bool `json:"replace"`
|
||||||
}
|
}
|
||||||
if err := decodeJSON(r, &req); err != nil {
|
if err := decodeJSON(r, &req); err != nil {
|
||||||
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
||||||
@@ -44,7 +50,10 @@ func handleCreateSchedule(db *sql.DB) http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// All-or-nothing: if any date already has an assignment, reject the whole request.
|
// All-or-nothing, in both directions: without replace, one taken date
|
||||||
|
// rejects the whole request; with it, either every date moves or none
|
||||||
|
// does. The rota must never be left with a hole where a shift used to
|
||||||
|
// be, so the delete and the insert share one transaction.
|
||||||
tx, err := db.BeginTx(r.Context(), nil)
|
tx, err := db.BeginTx(r.Context(), nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
@@ -53,10 +62,18 @@ func handleCreateSchedule(db *sql.DB) http.HandlerFunc {
|
|||||||
defer tx.Rollback()
|
defer tx.Rollback()
|
||||||
|
|
||||||
for _, d := range req.Dates {
|
for _, d := range req.Dates {
|
||||||
|
if req.Replace {
|
||||||
|
if _, err := tx.ExecContext(r.Context(),
|
||||||
|
"DELETE FROM schedule_entries WHERE date = ?", d); err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
if _, err := tx.ExecContext(r.Context(),
|
if _, err := tx.ExecContext(r.Context(),
|
||||||
"INSERT INTO schedule_entries (user_id, date) VALUES (?, ?)", req.UserID, d); err != nil {
|
"INSERT INTO schedule_entries (user_id, date) VALUES (?, ?)", req.UserID, d); err != nil {
|
||||||
if strings.Contains(err.Error(), "UNIQUE constraint failed") {
|
if strings.Contains(err.Error(), "UNIQUE constraint failed") {
|
||||||
respond(w, http.StatusConflict, errResp("date already assigned: "+d))
|
respond(w, http.StatusConflict,
|
||||||
|
errResp("date already assigned: "+d+" (pass replace to take it)"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
|||||||
+58
-14
@@ -11,13 +11,16 @@ import (
|
|||||||
|
|
||||||
func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
|
func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
where, args := statsFilter(r.URL.Query())
|
where, args := statsFilter(r.URL.Query(), "received_at")
|
||||||
|
|
||||||
|
// COALESCE because SUM over zero rows is NULL, not 0, and a count of
|
||||||
|
// nothing is 0 — without it an empty window is a 500 rather than a
|
||||||
|
// legitimately empty report.
|
||||||
var total, firing, resolved int64
|
var total, firing, resolved int64
|
||||||
err := db.QueryRowContext(r.Context(), fmt.Sprintf(`
|
err := db.QueryRowContext(r.Context(), fmt.Sprintf(`
|
||||||
SELECT COUNT(*),
|
SELECT COUNT(*),
|
||||||
SUM(CASE WHEN status = 'firing' THEN 1 ELSE 0 END),
|
COALESCE(SUM(CASE WHEN status = 'firing' THEN 1 ELSE 0 END), 0),
|
||||||
SUM(CASE WHEN status = 'resolved' THEN 1 ELSE 0 END)
|
COALESCE(SUM(CASE WHEN status = 'resolved' THEN 1 ELSE 0 END), 0)
|
||||||
FROM alerts WHERE %s`, where), args...,
|
FROM alerts WHERE %s`, where), args...,
|
||||||
).Scan(&total, &firing, &resolved)
|
).Scan(&total, &firing, &resolved)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -34,7 +37,7 @@ func handleStatsAlerts(db *sql.DB) http.HandlerFunc {
|
|||||||
|
|
||||||
func handleStatsTop(db *sql.DB) http.HandlerFunc {
|
func handleStatsTop(db *sql.DB) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
where, args := statsFilter(r.URL.Query())
|
where, args := statsFilter(r.URL.Query(), "received_at")
|
||||||
|
|
||||||
limit := 10
|
limit := 10
|
||||||
if l := r.URL.Query().Get("limit"); l != "" {
|
if l := r.URL.Query().Get("limit"); l != "" {
|
||||||
@@ -78,7 +81,7 @@ func handleStatsTop(db *sql.DB) http.HandlerFunc {
|
|||||||
|
|
||||||
func handleStatsByHour(db *sql.DB) http.HandlerFunc {
|
func handleStatsByHour(db *sql.DB) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
where, args := statsFilter(r.URL.Query())
|
where, args := statsFilter(r.URL.Query(), "received_at")
|
||||||
|
|
||||||
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
||||||
SELECT CAST(strftime('%%H', datetime(received_at, 'unixepoch')) AS INTEGER) AS hr,
|
SELECT CAST(strftime('%%H', datetime(received_at, 'unixepoch')) AS INTEGER) AS hr,
|
||||||
@@ -118,7 +121,7 @@ func handleStatsByHour(db *sql.DB) http.HandlerFunc {
|
|||||||
|
|
||||||
func handleStatsByDay(db *sql.DB) http.HandlerFunc {
|
func handleStatsByDay(db *sql.DB) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
where, args := statsFilter(r.URL.Query())
|
where, args := statsFilter(r.URL.Query(), "received_at")
|
||||||
|
|
||||||
// SQLite strftime('%w') → 0=Sunday … 6=Saturday
|
// SQLite strftime('%w') → 0=Sunday … 6=Saturday
|
||||||
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
rows, err := db.QueryContext(r.Context(), fmt.Sprintf(`
|
||||||
@@ -159,23 +162,64 @@ func handleStatsByDay(db *sql.DB) http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// statsFilter builds a WHERE clause and args from optional ?from and ?to query params.
|
// handleStatsIncidents reports the queue and the two numbers a rota actually
|
||||||
func statsFilter(q url.Values) (where string, args []any) {
|
// cares about: how long it takes someone to pick work up, and how long it takes
|
||||||
clauses := []string{}
|
// to finish. Neither was computable before incidents existed — alert rows are
|
||||||
|
// mutated in place and carry no acknowledgement or closure time.
|
||||||
|
func handleStatsIncidents(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
where, args := statsFilter(r.URL.Query(), "triggered_at")
|
||||||
|
|
||||||
|
// The counts are COALESCEd because SUM over zero rows is NULL, not 0.
|
||||||
|
// The averages are not: mtta and mttr stay null on purpose, since zero
|
||||||
|
// would read as "instant" rather than "nothing to measure yet".
|
||||||
|
var total, triggered, acknowledged, resolved int64
|
||||||
|
var mtta, mttr *float64
|
||||||
|
err := db.QueryRowContext(r.Context(), fmt.Sprintf(`
|
||||||
|
SELECT COUNT(*),
|
||||||
|
COALESCE(SUM(CASE WHEN status = 'triggered' THEN 1 ELSE 0 END), 0),
|
||||||
|
COALESCE(SUM(CASE WHEN status = 'acknowledged' THEN 1 ELSE 0 END), 0),
|
||||||
|
COALESCE(SUM(CASE WHEN status = 'resolved' THEN 1 ELSE 0 END), 0),
|
||||||
|
AVG(CASE WHEN acknowledged_at IS NOT NULL
|
||||||
|
THEN acknowledged_at - triggered_at END),
|
||||||
|
AVG(CASE WHEN resolved_at IS NOT NULL
|
||||||
|
THEN resolved_at - triggered_at END)
|
||||||
|
FROM incidents WHERE %s`, where), args...,
|
||||||
|
).Scan(&total, &triggered, &acknowledged, &resolved, &mtta, &mttr)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
respond(w, http.StatusOK, map[string]any{
|
||||||
|
"total": total,
|
||||||
|
"triggered": triggered,
|
||||||
|
"acknowledged": acknowledged,
|
||||||
|
"resolved": resolved,
|
||||||
|
// Null until something has actually been acknowledged or resolved —
|
||||||
|
// zero would read as "instant", which is a different claim.
|
||||||
|
"mtta_seconds": mtta,
|
||||||
|
"mttr_seconds": mttr,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// statsFilter builds a WHERE clause and args from optional ?from and ?to query
|
||||||
|
// params, filtering on timeCol. Archived rows are always excluded, matching the
|
||||||
|
// default list views.
|
||||||
|
func statsFilter(q url.Values, timeCol string) (where string, args []any) {
|
||||||
|
clauses := []string{"archived_at IS NULL"}
|
||||||
if from := q.Get("from"); from != "" {
|
if from := q.Get("from"); from != "" {
|
||||||
if t, err := time.Parse("2006-01-02", from); err == nil {
|
if t, err := time.Parse("2006-01-02", from); err == nil {
|
||||||
clauses = append(clauses, "received_at >= ?")
|
clauses = append(clauses, timeCol+" >= ?")
|
||||||
args = append(args, t.UTC().Unix())
|
args = append(args, t.UTC().Unix())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if to := q.Get("to"); to != "" {
|
if to := q.Get("to"); to != "" {
|
||||||
if t, err := time.Parse("2006-01-02", to); err == nil {
|
if t, err := time.Parse("2006-01-02", to); err == nil {
|
||||||
clauses = append(clauses, "received_at < ?")
|
clauses = append(clauses, timeCol+" < ?")
|
||||||
args = append(args, t.UTC().AddDate(0, 0, 1).Unix())
|
args = append(args, t.UTC().AddDate(0, 0, 1).Unix())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(clauses) == 0 {
|
|
||||||
return "1=1", args
|
|
||||||
}
|
|
||||||
return strings.Join(clauses, " AND "), args
|
return strings.Join(clauses, " AND "), args
|
||||||
}
|
}
|
||||||
|
|||||||
+73
-10
@@ -11,8 +11,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.ryuvia.com/niklas/terdut-server/internal/models"
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/yeniklas/terdut-server/internal/models"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func handleBootstrap(db *sql.DB) http.HandlerFunc {
|
func handleBootstrap(db *sql.DB) http.HandlerFunc {
|
||||||
@@ -20,6 +20,9 @@ func handleBootstrap(db *sql.DB) http.HandlerFunc {
|
|||||||
var req struct {
|
var req struct {
|
||||||
Username string `json:"username"`
|
Username string `json:"username"`
|
||||||
Email string `json:"email"`
|
Email string `json:"email"`
|
||||||
|
// Password is optional; without one the first user can only use the
|
||||||
|
// API key until somebody sets it.
|
||||||
|
Password string `json:"password"`
|
||||||
}
|
}
|
||||||
if err := decodeJSON(r, &req); err != nil {
|
if err := decodeJSON(r, &req); err != nil {
|
||||||
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
||||||
@@ -29,6 +32,19 @@ func handleBootstrap(db *sql.DB) http.HandlerFunc {
|
|||||||
respond(w, http.StatusBadRequest, errResp("username and email are required"))
|
respond(w, http.StatusBadRequest, errResp("username and email are required"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
var passwordHash *string
|
||||||
|
if req.Password != "" {
|
||||||
|
if msg := validatePassword(req.Password); msg != "" {
|
||||||
|
respond(w, http.StatusBadRequest, errResp(msg))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h, err := hashPassword(req.Password)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
passwordHash = &h
|
||||||
|
}
|
||||||
|
|
||||||
var count int
|
var count int
|
||||||
if err := db.QueryRowContext(r.Context(), "SELECT COUNT(*) FROM users").Scan(&count); err != nil {
|
if err := db.QueryRowContext(r.Context(), "SELECT COUNT(*) FROM users").Scan(&count); err != nil {
|
||||||
@@ -41,14 +57,15 @@ func handleBootstrap(db *sql.DB) http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
|
|
||||||
res, err := db.ExecContext(r.Context(),
|
res, err := db.ExecContext(r.Context(),
|
||||||
"INSERT INTO users (username, email) VALUES (?, ?)", req.Username, req.Email)
|
"INSERT INTO users (username, email, password_hash) VALUES (?, ?, ?)",
|
||||||
|
req.Username, req.Email, passwordHash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
userID, _ := res.LastInsertId()
|
userID, _ := res.LastInsertId()
|
||||||
|
|
||||||
raw, hash, err := newAPIKey()
|
raw, hash, err := randomToken()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
return
|
return
|
||||||
@@ -70,7 +87,7 @@ func handleBootstrap(db *sql.DB) http.HandlerFunc {
|
|||||||
func handleListUsers(db *sql.DB) http.HandlerFunc {
|
func handleListUsers(db *sql.DB) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
rows, err := db.QueryContext(r.Context(),
|
rows, err := db.QueryContext(r.Context(),
|
||||||
"SELECT id, username, email, created_at FROM users ORDER BY id")
|
"SELECT id, username, email, created_at, ntfy_topic FROM users ORDER BY id")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
return
|
return
|
||||||
@@ -81,7 +98,7 @@ func handleListUsers(db *sql.DB) http.HandlerFunc {
|
|||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var u models.User
|
var u models.User
|
||||||
var ts int64
|
var ts int64
|
||||||
if err := rows.Scan(&u.ID, &u.Username, &u.Email, &ts); err != nil {
|
if err := rows.Scan(&u.ID, &u.Username, &u.Email, &ts, &u.NtfyTopic); err != nil {
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -123,6 +140,50 @@ func handleCreateUser(db *sql.DB) http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handleSetNotifyTarget points a user's push notifications at an ntfy topic, or
|
||||||
|
// clears it with an empty string. The topic is a shared secret with the ntfy
|
||||||
|
// server — anyone who knows it can publish to it — so pick an unguessable one
|
||||||
|
// unless your ntfy enforces access control.
|
||||||
|
func handleSetNotifyTarget(db *sql.DB) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid user id"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var req struct {
|
||||||
|
NtfyTopic string `json:"ntfy_topic"`
|
||||||
|
}
|
||||||
|
if err := decodeJSON(r, &req); err != nil {
|
||||||
|
respond(w, http.StatusBadRequest, errResp("invalid request body"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var topic *string
|
||||||
|
if t := strings.TrimSpace(req.NtfyTopic); t != "" {
|
||||||
|
topic = &t
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := db.ExecContext(r.Context(),
|
||||||
|
"UPDATE users SET ntfy_topic = ? WHERE id = ?", topic, id)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if n, _ := res.RowsAffected(); n == 0 {
|
||||||
|
respond(w, http.StatusNotFound, errResp("user not found"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
user, err := fetchUser(r.Context(), db, id)
|
||||||
|
if err != nil {
|
||||||
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
respond(w, http.StatusOK, user)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func handleDeleteUser(db *sql.DB) http.HandlerFunc {
|
func handleDeleteUser(db *sql.DB) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
id, err := strconv.ParseInt(chi.URLParam(r, "id"), 10, 64)
|
||||||
@@ -170,7 +231,7 @@ func handleCreateAPIKey(db *sql.DB) http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
raw, hash, err := newAPIKey()
|
raw, hash, err := randomToken()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
respond(w, http.StatusInternalServerError, errResp("internal error"))
|
||||||
return
|
return
|
||||||
@@ -215,8 +276,9 @@ func handleDeleteAPIKey(db *sql.DB) http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// newAPIKey generates a random 32-byte key encoded as hex, plus its SHA-256 hash for storage.
|
// randomToken generates a random 32-byte secret encoded as hex, plus its SHA-256
|
||||||
func newAPIKey() (raw, hash string, err error) {
|
// hash for storage. Used for API keys and for notification acknowledge tokens.
|
||||||
|
func randomToken() (raw, hash string, err error) {
|
||||||
b := make([]byte, 32)
|
b := make([]byte, 32)
|
||||||
if _, err = rand.Read(b); err != nil {
|
if _, err = rand.Read(b); err != nil {
|
||||||
return
|
return
|
||||||
@@ -230,8 +292,9 @@ func newAPIKey() (raw, hash string, err error) {
|
|||||||
func fetchUser(ctx context.Context, db *sql.DB, id int64) (models.User, error) {
|
func fetchUser(ctx context.Context, db *sql.DB, id int64) (models.User, error) {
|
||||||
var u models.User
|
var u models.User
|
||||||
var ts int64
|
var ts int64
|
||||||
err := db.QueryRowContext(ctx, "SELECT id, username, email, created_at FROM users WHERE id = ?", id).
|
err := db.QueryRowContext(ctx,
|
||||||
Scan(&u.ID, &u.Username, &u.Email, &ts)
|
"SELECT id, username, email, created_at, ntfy_topic FROM users WHERE id = ?", id).
|
||||||
|
Scan(&u.ID, &u.Username, &u.Email, &ts, &u.NtfyTopic)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return u, err
|
return u, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,49 @@ type Config struct {
|
|||||||
Addr string
|
Addr string
|
||||||
DBPath string
|
DBPath string
|
||||||
ArchiveAfter time.Duration
|
ArchiveAfter time.Duration
|
||||||
|
|
||||||
|
// StaleAfter is how long a firing alert may go without a refreshing webhook
|
||||||
|
// before the sweeper treats it as resolved. It must exceed Alertmanager's
|
||||||
|
// repeat_interval (default 4h), which is what refreshes the alert.
|
||||||
|
StaleAfter time.Duration
|
||||||
|
|
||||||
|
// DeadmanMatchers selects the alerts that are heartbeats rather than
|
||||||
|
// problems: receiving one opens no incident, and the absence of one does.
|
||||||
|
//
|
||||||
|
// ";" separates matchers, "," the label conditions within one, "=" is exact
|
||||||
|
// equality — `alertname=Watchdog,cluster=prod; alertname=Heartbeat`. Every
|
||||||
|
// matcher must name an alertname. See api.ParseDeadmanConfig.
|
||||||
|
DeadmanMatchers string
|
||||||
|
|
||||||
|
// DeadmanTimeout is how long a heartbeat may go unheard before its switch is
|
||||||
|
// declared dead. It must be *shorter* than the Alertmanager repeat_interval
|
||||||
|
// of the route carrying the heartbeat — the opposite of StaleAfter, and the
|
||||||
|
// reason a dead man's switch usually wants a route of its own. Zero disables
|
||||||
|
// dead man's switch handling entirely.
|
||||||
|
DeadmanTimeout time.Duration
|
||||||
|
|
||||||
|
// DeadmanSeverity is the severity a dead man's switch incident opens at.
|
||||||
|
// These incidents have no member alerts to derive one from.
|
||||||
|
DeadmanSeverity string
|
||||||
|
|
||||||
|
// NtfyURL is the ntfy server push notifications are published to. Empty
|
||||||
|
// disables notifications entirely.
|
||||||
|
NtfyURL string
|
||||||
|
|
||||||
|
// NtfyToken is an optional bearer token for an access-controlled ntfy.
|
||||||
|
NtfyToken string
|
||||||
|
|
||||||
|
// NtfyFallbackTopic receives incidents that open with nobody on call.
|
||||||
|
NtfyFallbackTopic string
|
||||||
|
|
||||||
|
// PublicURL is the base URL a phone uses to reach this server, used for the
|
||||||
|
// link and the Acknowledge button inside a notification. Without it
|
||||||
|
// notifications carry neither.
|
||||||
|
PublicURL string
|
||||||
|
|
||||||
|
// NotifyRepeat is how long an incident may sit unacknowledged before it is
|
||||||
|
// notified again. Zero disables reminders.
|
||||||
|
NotifyRepeat time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
func Load() Config {
|
func Load() Config {
|
||||||
@@ -20,11 +63,40 @@ func Load() Config {
|
|||||||
if dbPath == "" {
|
if dbPath == "" {
|
||||||
dbPath = "terdut.db"
|
dbPath = "terdut.db"
|
||||||
}
|
}
|
||||||
archiveAfter := 7 * 24 * time.Hour
|
deadmanMatchers := os.Getenv("TERDUT_DEADMAN_MATCHERS")
|
||||||
if s := os.Getenv("TERDUT_ARCHIVE_AFTER"); s != "" {
|
if deadmanMatchers == "" {
|
||||||
|
deadmanMatchers = "alertname=Watchdog"
|
||||||
|
}
|
||||||
|
deadmanSeverity := os.Getenv("TERDUT_DEADMAN_SEVERITY")
|
||||||
|
if deadmanSeverity == "" {
|
||||||
|
deadmanSeverity = "critical"
|
||||||
|
}
|
||||||
|
return Config{
|
||||||
|
Addr: addr,
|
||||||
|
DBPath: dbPath,
|
||||||
|
ArchiveAfter: duration("TERDUT_ARCHIVE_AFTER", 7*24*time.Hour),
|
||||||
|
StaleAfter: duration("TERDUT_STALE_AFTER", 6*time.Hour),
|
||||||
|
|
||||||
|
DeadmanMatchers: deadmanMatchers,
|
||||||
|
DeadmanTimeout: duration("TERDUT_DEADMAN_TIMEOUT", 15*time.Minute),
|
||||||
|
DeadmanSeverity: deadmanSeverity,
|
||||||
|
|
||||||
|
NtfyURL: os.Getenv("TERDUT_NTFY_URL"),
|
||||||
|
NtfyToken: os.Getenv("TERDUT_NTFY_TOKEN"),
|
||||||
|
NtfyFallbackTopic: os.Getenv("TERDUT_NTFY_FALLBACK_TOPIC"),
|
||||||
|
PublicURL: os.Getenv("TERDUT_PUBLIC_URL"),
|
||||||
|
NotifyRepeat: duration("TERDUT_NOTIFY_REPEAT", 15*time.Minute),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// duration reads a time.ParseDuration-formatted env var. An unset or
|
||||||
|
// unparseable value falls back to def rather than failing startup: a typo in one
|
||||||
|
// tuning knob should not take the server down.
|
||||||
|
func duration(env string, def time.Duration) time.Duration {
|
||||||
|
if s := os.Getenv(env); s != "" {
|
||||||
if d, err := time.ParseDuration(s); err == nil {
|
if d, err := time.ParseDuration(s); err == nil {
|
||||||
archiveAfter = d
|
return d
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return Config{Addr: addr, DBPath: dbPath, ArchiveAfter: archiveAfter}
|
return def
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
-- Records why an alert left the firing state: 'alertmanager' when a resolved
|
||||||
|
-- webhook set it, 'expiry' when the sweeper inferred it from staleness.
|
||||||
|
-- NULL for firing alerts and for rows that predate this migration.
|
||||||
|
ALTER TABLE alerts ADD COLUMN resolution_source TEXT;
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
-- Splits the single alerts row into two objects, the way an incident management
|
||||||
|
-- tool needs them: alerts stay the machine-owned signal record that Alertmanager
|
||||||
|
-- writes, and incidents become the human work item people acknowledge, assign,
|
||||||
|
-- snooze, discuss and resolve.
|
||||||
|
--
|
||||||
|
-- Correlation uses Alertmanager's own groupKey, so incidents follow the group_by
|
||||||
|
-- routing tree the operator already tuned rather than a second grouping scheme
|
||||||
|
-- invented here.
|
||||||
|
|
||||||
|
CREATE TABLE incidents (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
group_key TEXT NOT NULL, -- Alertmanager groupKey, opaque
|
||||||
|
title TEXT NOT NULL, -- rendered from group_labels
|
||||||
|
group_labels TEXT NOT NULL DEFAULT '{}', -- JSON
|
||||||
|
status TEXT NOT NULL CHECK(status IN ('triggered', 'acknowledged', 'resolved')),
|
||||||
|
severity TEXT, -- highest `severity` label across firing members
|
||||||
|
triggered_at INTEGER NOT NULL,
|
||||||
|
acknowledged_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||||
|
acknowledged_at INTEGER,
|
||||||
|
assigned_to INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||||
|
snoozed_until INTEGER,
|
||||||
|
resolved_at INTEGER,
|
||||||
|
resolution_source TEXT, -- 'alerts' | 'manual'
|
||||||
|
archived_at INTEGER
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Load-bearing: at most one OPEN incident per group_key. This is what makes
|
||||||
|
-- "resolved incident + a new alert occurrence = a new incident" work, and it is
|
||||||
|
-- the constraint the webhook's find-or-open lookup relies on.
|
||||||
|
CREATE UNIQUE INDEX incidents_open_group_key_idx ON incidents(group_key) WHERE resolved_at IS NULL;
|
||||||
|
CREATE INDEX incidents_status_idx ON incidents(status);
|
||||||
|
CREATE INDEX incidents_triggered_at_idx ON incidents(triggered_at DESC);
|
||||||
|
CREATE INDEX incidents_archived_at_idx ON incidents(archived_at);
|
||||||
|
|
||||||
|
-- Membership is historical, not a pointer on alerts: one alert row (one
|
||||||
|
-- fingerprint) resolves and re-fires over time and belongs to a different
|
||||||
|
-- incident each occurrence.
|
||||||
|
CREATE TABLE incident_alerts (
|
||||||
|
incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
|
||||||
|
alert_id INTEGER NOT NULL REFERENCES alerts(id) ON DELETE CASCADE,
|
||||||
|
added_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now')),
|
||||||
|
PRIMARY KEY (incident_id, alert_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX incident_alerts_alert_id_idx ON incident_alerts(alert_id);
|
||||||
|
|
||||||
|
-- The timeline. Append-only, and the only history this server keeps: alert rows
|
||||||
|
-- are mutated in place, so without this there is no record that anything
|
||||||
|
-- happened. Notes are events too, so one query renders the whole story.
|
||||||
|
CREATE TABLE incident_events (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
|
||||||
|
-- triggered | alert_added | alert_resolved | acknowledged | unacknowledged
|
||||||
|
-- | assigned | snoozed | unsnoozed | resolved | note
|
||||||
|
type TEXT NOT NULL,
|
||||||
|
user_id INTEGER REFERENCES users(id) ON DELETE SET NULL, -- NULL = the server acted
|
||||||
|
alert_id INTEGER REFERENCES alerts(id) ON DELETE SET NULL,
|
||||||
|
detail TEXT,
|
||||||
|
created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX incident_events_incident_idx ON incident_events(incident_id, created_at);
|
||||||
|
|
||||||
|
-- ---------------------------------------------------------------------------
|
||||||
|
-- Backfill
|
||||||
|
--
|
||||||
|
-- Every pre-existing alert gets its own incident, archived ones included, so no
|
||||||
|
-- acknowledgement and no comment is orphaned. There is no historical groupKey to
|
||||||
|
-- correlate on, hence one incident per fingerprint under a 'backfill:' prefix
|
||||||
|
-- that can never collide with a real Alertmanager groupKey.
|
||||||
|
-- ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
INSERT INTO incidents (group_key, title, group_labels, status, severity, triggered_at,
|
||||||
|
acknowledged_by, acknowledged_at, assigned_to,
|
||||||
|
resolved_at, resolution_source, archived_at)
|
||||||
|
SELECT 'backfill:' || a.fingerprint,
|
||||||
|
a.name,
|
||||||
|
json_object('alertname', a.name),
|
||||||
|
CASE WHEN a.status = 'resolved' THEN 'resolved'
|
||||||
|
WHEN a.acknowledged_by IS NOT NULL THEN 'acknowledged'
|
||||||
|
ELSE 'triggered' END,
|
||||||
|
json_extract(a.labels, '$.severity'),
|
||||||
|
a.starts_at,
|
||||||
|
a.acknowledged_by,
|
||||||
|
a.acknowledged_at,
|
||||||
|
a.acknowledged_by,
|
||||||
|
CASE WHEN a.status = 'resolved' THEN COALESCE(a.ends_at, a.received_at) END,
|
||||||
|
CASE WHEN a.status = 'resolved' THEN 'alerts' END,
|
||||||
|
a.archived_at
|
||||||
|
FROM alerts a;
|
||||||
|
|
||||||
|
INSERT INTO incident_alerts (incident_id, alert_id, added_at)
|
||||||
|
SELECT i.id, a.id, a.starts_at
|
||||||
|
FROM alerts a
|
||||||
|
JOIN incidents i ON i.group_key = 'backfill:' || a.fingerprint;
|
||||||
|
|
||||||
|
INSERT INTO incident_events (incident_id, type, alert_id, created_at)
|
||||||
|
SELECT i.id, 'triggered', ia.alert_id, i.triggered_at
|
||||||
|
FROM incidents i JOIN incident_alerts ia ON ia.incident_id = i.id;
|
||||||
|
|
||||||
|
INSERT INTO incident_events (incident_id, type, user_id, created_at)
|
||||||
|
SELECT i.id, 'acknowledged', i.acknowledged_by, i.acknowledged_at
|
||||||
|
FROM incidents i WHERE i.acknowledged_at IS NOT NULL;
|
||||||
|
|
||||||
|
INSERT INTO incident_events (incident_id, type, created_at)
|
||||||
|
SELECT i.id, 'resolved', i.resolved_at
|
||||||
|
FROM incidents i WHERE i.resolved_at IS NOT NULL;
|
||||||
|
|
||||||
|
INSERT INTO incident_events (incident_id, type, user_id, alert_id, detail, created_at)
|
||||||
|
SELECT ia.incident_id, 'note', c.user_id, c.alert_id, c.content, c.created_at
|
||||||
|
FROM alert_comments c
|
||||||
|
JOIN incident_alerts ia ON ia.alert_id = c.alert_id;
|
||||||
|
|
||||||
|
-- ---------------------------------------------------------------------------
|
||||||
|
-- Workflow state now lives on incidents only. Leaving these behind would keep
|
||||||
|
-- the bug they caused: the webhook upsert owns the alerts row and never cleared
|
||||||
|
-- the acknowledgement, so a re-fire days later still read as acknowledged.
|
||||||
|
-- ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
DROP TABLE alert_comments;
|
||||||
|
|
||||||
|
ALTER TABLE alerts DROP COLUMN acknowledged_by;
|
||||||
|
ALTER TABLE alerts DROP COLUMN acknowledged_at;
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
-- Adds push notification delivery, so an incident reaches the person on call
|
||||||
|
-- instead of waiting to be discovered.
|
||||||
|
--
|
||||||
|
-- Delivery is an outbox rather than an inline HTTP call: the pool is limited to
|
||||||
|
-- a single connection, so a POST made while holding the webhook's transaction
|
||||||
|
-- would stall every other request behind it. The webhook inserts a row; the
|
||||||
|
-- notifier goroutine delivers it.
|
||||||
|
|
||||||
|
CREATE TABLE notifications (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
|
||||||
|
-- Nullable: a notification sent to the fallback topic belongs to nobody,
|
||||||
|
-- because nobody was on call when the incident opened.
|
||||||
|
user_id INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||||
|
topic TEXT NOT NULL, -- resolved at enqueue: who was on call then
|
||||||
|
kind TEXT NOT NULL CHECK(kind IN ('triggered', 'reminder', 'resolved')),
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
send_after INTEGER NOT NULL, -- retry backoff watermark
|
||||||
|
attempts INTEGER NOT NULL DEFAULT 0,
|
||||||
|
sent_at INTEGER,
|
||||||
|
last_error TEXT -- kept after the last attempt, for debugging
|
||||||
|
);
|
||||||
|
|
||||||
|
-- The delivery loop's only query: what is due and still unsent.
|
||||||
|
CREATE INDEX notifications_pending_idx ON notifications(send_after) WHERE sent_at IS NULL;
|
||||||
|
-- Reminders and resolved notices both look up an incident's newest row.
|
||||||
|
CREATE INDEX notifications_incident_idx ON notifications(incident_id, id DESC);
|
||||||
|
|
||||||
|
-- A notification body is stored on the ntfy server and cached on the device, so
|
||||||
|
-- a real API key must never appear in one. Each delivery mints its own token
|
||||||
|
-- instead: one incident, one action, one day.
|
||||||
|
CREATE TABLE incident_ack_tokens (
|
||||||
|
token_hash TEXT PRIMARY KEY, -- SHA-256 of the raw token, as with api_keys
|
||||||
|
incident_id INTEGER NOT NULL REFERENCES incidents(id) ON DELETE CASCADE,
|
||||||
|
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
expires_at INTEGER NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX incident_ack_tokens_expires_idx ON incident_ack_tokens(expires_at);
|
||||||
|
|
||||||
|
-- Where this user's notifications go. NULL means they get none; incidents
|
||||||
|
-- assigned to them fall back to the configured fallback topic.
|
||||||
|
ALTER TABLE users ADD COLUMN ntfy_topic TEXT;
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
-- A password is what lets a person sign in to the web UI. NULL means the user
|
||||||
|
-- has none and can only use API keys, which is every user created before this.
|
||||||
|
ALTER TABLE users ADD COLUMN password_hash TEXT;
|
||||||
|
|
||||||
|
-- A session is a browser's credential, the cookie counterpart of an API key:
|
||||||
|
-- only the hash of the token is stored. expires_at slides forward while the
|
||||||
|
-- session is in use, so an on-call phone stays signed in.
|
||||||
|
CREATE TABLE sessions (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
token_hash TEXT NOT NULL UNIQUE,
|
||||||
|
user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||||
|
created_at INTEGER NOT NULL,
|
||||||
|
last_seen_at INTEGER NOT NULL,
|
||||||
|
expires_at INTEGER NOT NULL,
|
||||||
|
user_agent TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_sessions_user ON sessions(user_id);
|
||||||
@@ -2,6 +2,10 @@ package models
|
|||||||
|
|
||||||
import "time"
|
import "time"
|
||||||
|
|
||||||
|
// Alert is the machine-owned signal record: what Alertmanager told us, and
|
||||||
|
// nothing else. It has two states, firing and resolved, and no human ever writes
|
||||||
|
// to it — acknowledgement, assignment, notes and closure all live on the
|
||||||
|
// Incident an alert belongs to.
|
||||||
type Alert struct {
|
type Alert struct {
|
||||||
ID int64 `json:"id"`
|
ID int64 `json:"id"`
|
||||||
Fingerprint string `json:"fingerprint"`
|
Fingerprint string `json:"fingerprint"`
|
||||||
@@ -12,12 +16,39 @@ type Alert struct {
|
|||||||
StartsAt time.Time `json:"starts_at"`
|
StartsAt time.Time `json:"starts_at"`
|
||||||
EndsAt *time.Time `json:"ends_at,omitempty"`
|
EndsAt *time.Time `json:"ends_at,omitempty"`
|
||||||
GeneratorURL string `json:"generator_url"`
|
GeneratorURL string `json:"generator_url"`
|
||||||
ReceivedAt time.Time `json:"received_at"`
|
|
||||||
|
|
||||||
// Populated when the alert has been acknowledged.
|
// ReceivedAt is when the server last accepted a webhook for this
|
||||||
AcknowledgedByID *int64 `json:"acknowledged_by_id,omitempty"`
|
// fingerprint, including the unchanged firing notifications Alertmanager
|
||||||
AcknowledgedByUser *string `json:"acknowledged_by,omitempty"`
|
// re-sends every repeat_interval.
|
||||||
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
|
//
|
||||||
|
// This is a documented part of the public API, not an internal ingest
|
||||||
|
// detail: StartsAt never changes for an alert instance, so ReceivedAt is
|
||||||
|
// the only signal a client has that a firing alert is still being
|
||||||
|
// refreshed. The sweeper stale-dates against it (see expireStale), API
|
||||||
|
// clients render it, and GET /api/alerts is ordered by it. Anything that
|
||||||
|
// stops the webhook handler from advancing it on a re-send is a breaking
|
||||||
|
// change — see "received_at is a liveness heartbeat" in the README and
|
||||||
|
// TestWebhook_ResendBumpsReceivedAt.
|
||||||
|
ReceivedAt time.Time `json:"received_at"`
|
||||||
|
|
||||||
|
// IncidentID is the most recent incident this alert belongs to. An alert row
|
||||||
|
// is reused across occurrences of the same fingerprint, so over its life it
|
||||||
|
// belongs to a series of incidents; incident_alerts keeps the full history
|
||||||
|
// and this is only the newest link.
|
||||||
|
IncidentID *int64 `json:"incident_id,omitempty"`
|
||||||
|
|
||||||
|
// ResolutionSource records why a resolved alert left the firing state:
|
||||||
|
// "alertmanager" for a real resolved webhook, "expiry" when the sweeper
|
||||||
|
// inferred it after the alert stopped being refreshed. Nil while firing, and
|
||||||
|
// cleared again by a re-fire under the same fingerprint.
|
||||||
|
//
|
||||||
|
// Also public API: it is how a client knows whether EndsAt was observed or
|
||||||
|
// inferred. Under "expiry" nothing ever reported an end, so EndsAt is only
|
||||||
|
// an upper bound (see expireStale) and ReceivedAt is the more truthful
|
||||||
|
// signal. Treat the value set as open — see "resolution_source says how much
|
||||||
|
// to trust ends_at" in the README, and TestWebhook_ResolvedSetsSource /
|
||||||
|
// TestExpiry_StaleFiringAlert.
|
||||||
|
ResolutionSource *string `json:"resolution_source,omitempty"`
|
||||||
|
|
||||||
ArchivedAt *time.Time `json:"archived_at,omitempty"`
|
ArchivedAt *time.Time `json:"archived_at,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +0,0 @@
|
|||||||
package models
|
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
type Comment struct {
|
|
||||||
ID int64 `json:"id"`
|
|
||||||
AlertID int64 `json:"alert_id"`
|
|
||||||
UserID int64 `json:"user_id"`
|
|
||||||
Username string `json:"username"`
|
|
||||||
Content string `json:"content"`
|
|
||||||
CreatedAt time.Time `json:"created_at"`
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
package models
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// Incident is the human work item: the thing that gets acknowledged, assigned,
|
||||||
|
// snoozed, discussed and resolved. Alerts are the machine-owned signal records
|
||||||
|
// underneath it — many alerts map to one incident, correlated by the groupKey
|
||||||
|
// Alertmanager already computed from the operator's group_by configuration.
|
||||||
|
//
|
||||||
|
// Nothing here is ever written by the Alertmanager webhook except Status, which
|
||||||
|
// the webhook and the sweeper may flip to "resolved" once every member alert has
|
||||||
|
// stopped firing.
|
||||||
|
type Incident struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
GroupKey string `json:"group_key"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
GroupLabels map[string]string `json:"group_labels"`
|
||||||
|
|
||||||
|
// Status is "triggered", "acknowledged" or "resolved".
|
||||||
|
Status string `json:"status"`
|
||||||
|
|
||||||
|
// Severity is the highest `severity` label across the alerts that were
|
||||||
|
// firing when it was last recomputed. It is deliberately not cleared when an
|
||||||
|
// incident resolves — a resolved incident should still say how bad it was.
|
||||||
|
Severity *string `json:"severity,omitempty"`
|
||||||
|
|
||||||
|
TriggeredAt time.Time `json:"triggered_at"`
|
||||||
|
|
||||||
|
AcknowledgedByID *int64 `json:"acknowledged_by_id,omitempty"`
|
||||||
|
AcknowledgedByUser *string `json:"acknowledged_by,omitempty"`
|
||||||
|
AcknowledgedAt *time.Time `json:"acknowledged_at,omitempty"`
|
||||||
|
|
||||||
|
AssignedToID *int64 `json:"assigned_to_id,omitempty"`
|
||||||
|
AssignedToUser *string `json:"assigned_to,omitempty"`
|
||||||
|
|
||||||
|
// SnoozedUntil hides the incident from the default queue without closing it.
|
||||||
|
// A timestamp in the past reads as "not snoozed"; nothing sweeps it.
|
||||||
|
SnoozedUntil *time.Time `json:"snoozed_until,omitempty"`
|
||||||
|
|
||||||
|
ResolvedAt *time.Time `json:"resolved_at,omitempty"`
|
||||||
|
|
||||||
|
// ResolutionSource is "alerts" when every member alert stopped firing, or
|
||||||
|
// "manual" when a human closed it. Manual resolution is terminal: a later
|
||||||
|
// occurrence opens a new incident rather than reopening this one.
|
||||||
|
ResolutionSource *string `json:"resolution_source,omitempty"`
|
||||||
|
|
||||||
|
ArchivedAt *time.Time `json:"archived_at,omitempty"`
|
||||||
|
|
||||||
|
// Alerts is populated by GET /api/incidents/{id} only.
|
||||||
|
Alerts []Alert `json:"alerts,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// IncidentEvent is one entry in an incident's timeline. The table is append-only
|
||||||
|
// and is the only history this server keeps — alert rows are mutated in place.
|
||||||
|
//
|
||||||
|
// Type is one of: triggered, alert_added, alert_resolved, acknowledged,
|
||||||
|
// unacknowledged, assigned, snoozed, unsnoozed, resolved, note. A nil UserID
|
||||||
|
// means the server acted rather than a person.
|
||||||
|
type IncidentEvent struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
IncidentID int64 `json:"incident_id"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
UserID *int64 `json:"user_id,omitempty"`
|
||||||
|
Username *string `json:"username,omitempty"`
|
||||||
|
AlertID *int64 `json:"alert_id,omitempty"`
|
||||||
|
Detail *string `json:"detail,omitempty"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
}
|
||||||
@@ -7,6 +7,11 @@ type User struct {
|
|||||||
Username string `json:"username"`
|
Username string `json:"username"`
|
||||||
Email string `json:"email"`
|
Email string `json:"email"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
|
||||||
|
// NtfyTopic is where this user's push notifications go. Nil means they get
|
||||||
|
// none of their own; incidents assigned to them fall back to the configured
|
||||||
|
// fallback topic instead.
|
||||||
|
NtfyTopic *string `json:"ntfy_topic,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type APIKey struct {
|
type APIKey struct {
|
||||||
|
|||||||
@@ -0,0 +1,622 @@
|
|||||||
|
/* terdut web UI.
|
||||||
|
*
|
||||||
|
* Mobile first: one column, a top bar and a bottom tab bar. From 900px the tab
|
||||||
|
* bar becomes a sidebar and the queue shows list and detail side by side.
|
||||||
|
* Colour is reserved for severity and status; everything else is neutral.
|
||||||
|
*/
|
||||||
|
|
||||||
|
:root {
|
||||||
|
--bg: #f5f6f8;
|
||||||
|
--surface: #ffffff;
|
||||||
|
--surface-2: #eff1f4;
|
||||||
|
--surface-hover: #f7f8fa;
|
||||||
|
--border: #e2e5ea;
|
||||||
|
--border-strong: #cfd3da;
|
||||||
|
--text: #16181d;
|
||||||
|
--muted: #5b626e;
|
||||||
|
--faint: #8a909b;
|
||||||
|
|
||||||
|
--accent: #2f5bd3;
|
||||||
|
--accent-text: #ffffff;
|
||||||
|
--accent-soft: #e8eefc;
|
||||||
|
|
||||||
|
--crit: #d0342c;
|
||||||
|
--crit-soft: #fdecea;
|
||||||
|
--warn: #b86e00;
|
||||||
|
--warn-soft: #fdf3e1;
|
||||||
|
--info: #2f6fdf;
|
||||||
|
--info-soft: #e9f0fd;
|
||||||
|
--ok: #1d7f4c;
|
||||||
|
--ok-soft: #e6f5ec;
|
||||||
|
--snooze: #6b5bd2;
|
||||||
|
--snooze-soft: #efedfb;
|
||||||
|
|
||||||
|
--radius: 10px;
|
||||||
|
--radius-sm: 6px;
|
||||||
|
--shadow: 0 1px 2px rgb(16 24 40 / 6%), 0 1px 3px rgb(16 24 40 / 8%);
|
||||||
|
--shadow-lg: 0 12px 32px rgb(16 24 40 / 18%);
|
||||||
|
|
||||||
|
--font: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
|
||||||
|
--mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||||
|
|
||||||
|
--topbar-h: 52px;
|
||||||
|
--tabbar-h: 58px;
|
||||||
|
--safe-top: env(safe-area-inset-top, 0px);
|
||||||
|
--safe-bottom: env(safe-area-inset-bottom, 0px);
|
||||||
|
}
|
||||||
|
|
||||||
|
@media (prefers-color-scheme: dark) {
|
||||||
|
:root {
|
||||||
|
--bg: #0f1115;
|
||||||
|
--surface: #171a20;
|
||||||
|
--surface-2: #1f232b;
|
||||||
|
--surface-hover: #1c2027;
|
||||||
|
--border: #2a2f38;
|
||||||
|
--border-strong: #394050;
|
||||||
|
--text: #e7e9ed;
|
||||||
|
--muted: #a0a7b3;
|
||||||
|
--faint: #737a87;
|
||||||
|
|
||||||
|
--accent: #6d8ff0;
|
||||||
|
--accent-text: #0b0d12;
|
||||||
|
--accent-soft: #1d2640;
|
||||||
|
|
||||||
|
--crit: #ff6b61;
|
||||||
|
--crit-soft: #3a1c1b;
|
||||||
|
--warn: #f0b140;
|
||||||
|
--warn-soft: #362a14;
|
||||||
|
--info: #74a3ff;
|
||||||
|
--info-soft: #1a2640;
|
||||||
|
--ok: #4cc488;
|
||||||
|
--ok-soft: #15301f;
|
||||||
|
--snooze: #a89bff;
|
||||||
|
--snooze-soft: #262245;
|
||||||
|
|
||||||
|
--shadow: 0 1px 2px rgb(0 0 0 / 40%);
|
||||||
|
--shadow-lg: 0 16px 40px rgb(0 0 0 / 55%);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
*, *::before, *::after { box-sizing: border-box; }
|
||||||
|
[hidden] { display: none !important; }
|
||||||
|
|
||||||
|
html { -webkit-text-size-adjust: 100%; }
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
background: var(--bg);
|
||||||
|
color: var(--text);
|
||||||
|
font: 15px/1.45 var(--font);
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
-webkit-font-smoothing: antialiased;
|
||||||
|
-webkit-tap-highlight-color: transparent;
|
||||||
|
}
|
||||||
|
|
||||||
|
a { color: inherit; text-decoration: none; }
|
||||||
|
button, input, textarea, select { font: inherit; color: inherit; }
|
||||||
|
code { font-family: var(--mono); font-size: 0.9em; }
|
||||||
|
h1, h2, h3 { margin: 0; line-height: 1.25; }
|
||||||
|
|
||||||
|
:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
|
||||||
|
|
||||||
|
/* ---------- boot + login ---------- */
|
||||||
|
|
||||||
|
.boot { display: grid; place-items: center; min-height: 100dvh; }
|
||||||
|
.spinner {
|
||||||
|
width: 26px; height: 26px; border-radius: 50%;
|
||||||
|
border: 3px solid var(--border); border-top-color: var(--accent);
|
||||||
|
animation: spin 0.8s linear infinite;
|
||||||
|
}
|
||||||
|
@keyframes spin { to { transform: rotate(360deg); } }
|
||||||
|
|
||||||
|
.login {
|
||||||
|
min-height: 100dvh;
|
||||||
|
display: grid; place-items: center;
|
||||||
|
padding: calc(24px + var(--safe-top)) 16px calc(24px + var(--safe-bottom));
|
||||||
|
}
|
||||||
|
.login-card {
|
||||||
|
width: 100%; max-width: 360px;
|
||||||
|
display: grid; gap: 14px;
|
||||||
|
}
|
||||||
|
.login-brand { display: flex; align-items: center; gap: 10px; margin-bottom: 8px; }
|
||||||
|
.login-brand h1 { font-size: 24px; letter-spacing: -0.01em; }
|
||||||
|
.login-hint { color: var(--faint); font-size: 13px; margin: 4px 0 0; }
|
||||||
|
|
||||||
|
label { display: grid; gap: 6px; }
|
||||||
|
label > span { font-size: 13px; font-weight: 600; color: var(--muted); }
|
||||||
|
|
||||||
|
input, textarea, select {
|
||||||
|
width: 100%;
|
||||||
|
min-height: 44px;
|
||||||
|
padding: 10px 12px;
|
||||||
|
background: var(--surface);
|
||||||
|
border: 1px solid var(--border-strong);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
font-size: 16px; /* 16px stops iOS zooming into the field */
|
||||||
|
}
|
||||||
|
textarea { min-height: 110px; resize: vertical; line-height: 1.45; }
|
||||||
|
input:focus, textarea:focus { outline: none; border-color: var(--accent); box-shadow: 0 0 0 3px var(--accent-soft); }
|
||||||
|
|
||||||
|
.form-error {
|
||||||
|
margin: 0; padding: 10px 12px;
|
||||||
|
background: var(--crit-soft); color: var(--crit);
|
||||||
|
border-radius: var(--radius-sm); font-size: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---------- buttons ---------- */
|
||||||
|
|
||||||
|
.btn {
|
||||||
|
display: inline-flex; align-items: center; justify-content: center; gap: 8px;
|
||||||
|
min-height: 44px; padding: 0 16px;
|
||||||
|
border: 1px solid var(--border-strong);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
background: var(--surface);
|
||||||
|
font-weight: 600; font-size: 15px;
|
||||||
|
cursor: pointer;
|
||||||
|
white-space: nowrap;
|
||||||
|
transition: background 0.12s, border-color 0.12s, opacity 0.12s;
|
||||||
|
}
|
||||||
|
.btn:hover { background: var(--surface-hover); }
|
||||||
|
.btn:disabled { opacity: 0.55; cursor: default; }
|
||||||
|
.btn-primary { background: var(--accent); border-color: var(--accent); color: var(--accent-text); }
|
||||||
|
.btn-primary:hover { background: var(--accent); filter: brightness(1.06); }
|
||||||
|
.btn-danger { background: var(--crit); border-color: var(--crit); color: #fff; }
|
||||||
|
.btn-danger:hover { background: var(--crit); filter: brightness(1.06); }
|
||||||
|
.btn-ghost { background: transparent; border-color: transparent; }
|
||||||
|
.btn-ghost:hover { background: var(--surface-2); }
|
||||||
|
.btn-block { width: 100%; }
|
||||||
|
.btn-icon { width: 44px; padding: 0; }
|
||||||
|
.btn-sm { min-height: 32px; padding: 0 8px; font-size: 13px; }
|
||||||
|
.btn-sm svg { width: 16px; height: 16px; }
|
||||||
|
.btn svg, .icon { width: 20px; height: 20px; fill: none; stroke: currentColor; stroke-width: 2; stroke-linecap: round; stroke-linejoin: round; }
|
||||||
|
|
||||||
|
/* ---------- app frame ---------- */
|
||||||
|
|
||||||
|
.app { min-height: 100dvh; }
|
||||||
|
|
||||||
|
.topbar {
|
||||||
|
position: sticky; top: 0; z-index: 10;
|
||||||
|
display: flex; align-items: center; justify-content: space-between; gap: 12px;
|
||||||
|
height: calc(var(--topbar-h) + var(--safe-top));
|
||||||
|
padding: var(--safe-top) 16px 0;
|
||||||
|
background: color-mix(in srgb, var(--bg) 88%, transparent);
|
||||||
|
backdrop-filter: saturate(1.4) blur(12px);
|
||||||
|
-webkit-backdrop-filter: saturate(1.4) blur(12px);
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
.topbar-title { font-size: 18px; font-weight: 700; letter-spacing: -0.01em; }
|
||||||
|
|
||||||
|
.open-pill {
|
||||||
|
display: inline-flex; align-items: center; gap: 6px;
|
||||||
|
padding: 3px 10px; border-radius: 999px;
|
||||||
|
font-size: 13px; font-weight: 600;
|
||||||
|
background: var(--surface-2); color: var(--muted);
|
||||||
|
}
|
||||||
|
.open-pill::before { content: ""; width: 8px; height: 8px; border-radius: 50%; background: var(--faint); }
|
||||||
|
.open-pill.has-triggered { background: var(--crit-soft); color: var(--crit); }
|
||||||
|
.open-pill.has-triggered::before { background: var(--crit); }
|
||||||
|
.open-pill.all-acked::before { background: var(--warn); }
|
||||||
|
|
||||||
|
/* Bottom tab bar on phones. */
|
||||||
|
.nav {
|
||||||
|
position: fixed; left: 0; right: 0; bottom: 0; z-index: 20;
|
||||||
|
display: grid; grid-template-columns: repeat(4, 1fr);
|
||||||
|
height: calc(var(--tabbar-h) + var(--safe-bottom));
|
||||||
|
padding-bottom: var(--safe-bottom);
|
||||||
|
background: color-mix(in srgb, var(--surface) 92%, transparent);
|
||||||
|
backdrop-filter: saturate(1.4) blur(12px);
|
||||||
|
-webkit-backdrop-filter: saturate(1.4) blur(12px);
|
||||||
|
border-top: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
.nav-brand { display: none; }
|
||||||
|
.nav-link {
|
||||||
|
position: relative;
|
||||||
|
display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 2px;
|
||||||
|
color: var(--faint); font-size: 11px; font-weight: 600;
|
||||||
|
}
|
||||||
|
.nav-link svg { width: 24px; height: 24px; fill: none; stroke: currentColor; stroke-width: 1.8; stroke-linecap: round; stroke-linejoin: round; }
|
||||||
|
.nav-link[aria-current="page"] { color: var(--accent); }
|
||||||
|
.nav-badge {
|
||||||
|
position: absolute; top: 6px; left: calc(50% + 6px);
|
||||||
|
min-width: 18px; height: 18px; padding: 0 5px;
|
||||||
|
border-radius: 999px; background: var(--crit); color: #fff;
|
||||||
|
font-size: 11px; font-weight: 700; line-height: 18px; text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.view { padding-bottom: calc(var(--tabbar-h) + var(--safe-bottom)); }
|
||||||
|
.view-page { padding-left: 16px; padding-right: 16px; }
|
||||||
|
.view-page > * { max-width: 760px; margin-left: auto; margin-right: auto; }
|
||||||
|
|
||||||
|
/* Phone detail: the incident takes the whole screen with its own action bar,
|
||||||
|
so the tab bar and top bar step aside. */
|
||||||
|
.app.detail-open .nav,
|
||||||
|
.app.detail-open .topbar { display: none; }
|
||||||
|
.app.detail-open .pane-list { display: none; }
|
||||||
|
.app.detail-open .view-queue { padding-bottom: 0; }
|
||||||
|
.view-queue:not(.has-detail) .pane-detail { display: none; }
|
||||||
|
|
||||||
|
/* ---------- chips ---------- */
|
||||||
|
|
||||||
|
.chips {
|
||||||
|
display: flex; gap: 6px;
|
||||||
|
padding: 12px 16px 8px;
|
||||||
|
overflow-x: auto; scrollbar-width: none;
|
||||||
|
}
|
||||||
|
.chips::-webkit-scrollbar { display: none; }
|
||||||
|
.chip {
|
||||||
|
flex: none;
|
||||||
|
min-height: 34px; padding: 0 12px;
|
||||||
|
border: 1px solid var(--border-strong); border-radius: 999px;
|
||||||
|
background: var(--surface); color: var(--muted);
|
||||||
|
font-size: 13px; font-weight: 600; cursor: pointer;
|
||||||
|
}
|
||||||
|
.chip[aria-selected="true"] { background: var(--text); border-color: var(--text); color: var(--bg); }
|
||||||
|
.chip .count { margin-left: 4px; opacity: 0.7; }
|
||||||
|
|
||||||
|
/* ---------- lists ---------- */
|
||||||
|
|
||||||
|
.list { padding: 0 16px 16px; display: grid; gap: 8px; }
|
||||||
|
|
||||||
|
.row {
|
||||||
|
position: relative;
|
||||||
|
display: grid; grid-template-columns: 1fr auto; gap: 4px 12px;
|
||||||
|
padding: 11px 14px 11px 18px;
|
||||||
|
background: var(--surface);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
box-shadow: var(--shadow);
|
||||||
|
cursor: pointer;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
.row:hover { background: var(--surface-hover); }
|
||||||
|
.row[aria-current="true"] { border-color: var(--accent); box-shadow: 0 0 0 1px var(--accent); }
|
||||||
|
.row.kbd-focus { outline: 2px solid var(--accent); outline-offset: 1px; }
|
||||||
|
.row::before {
|
||||||
|
content: ""; position: absolute; left: 0; top: 0; bottom: 0; width: 4px;
|
||||||
|
background: var(--sev, var(--border-strong));
|
||||||
|
}
|
||||||
|
.row-title {
|
||||||
|
font-weight: 650; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||||
|
}
|
||||||
|
.row-age { color: var(--faint); font-size: 13px; text-align: right; white-space: nowrap; }
|
||||||
|
.row-meta {
|
||||||
|
grid-column: 1 / -1;
|
||||||
|
display: flex; flex-wrap: wrap; align-items: center; gap: 4px 8px;
|
||||||
|
color: var(--muted); font-size: 13px;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
.row-meta .labels { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; max-width: 100%; color: var(--faint); }
|
||||||
|
.row.resolved .row-title { color: var(--muted); }
|
||||||
|
|
||||||
|
.sev-critical { --sev: var(--crit); }
|
||||||
|
.sev-warning { --sev: var(--warn); }
|
||||||
|
.sev-info { --sev: var(--info); }
|
||||||
|
|
||||||
|
.empty {
|
||||||
|
padding: 48px 16px; text-align: center; color: var(--muted);
|
||||||
|
}
|
||||||
|
.empty strong { display: block; color: var(--text); font-size: 16px; margin-bottom: 4px; }
|
||||||
|
.empty .icon { width: 36px; height: 36px; color: var(--ok); margin-bottom: 8px; }
|
||||||
|
|
||||||
|
.load-error {
|
||||||
|
margin: 12px 16px; padding: 10px 12px;
|
||||||
|
background: var(--crit-soft); color: var(--crit);
|
||||||
|
border-radius: var(--radius-sm); font-size: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---------- badges ---------- */
|
||||||
|
|
||||||
|
.badge {
|
||||||
|
display: inline-flex; align-items: center; gap: 5px;
|
||||||
|
padding: 1px 8px; border-radius: 999px;
|
||||||
|
font-size: 12px; font-weight: 700; letter-spacing: 0.01em;
|
||||||
|
background: var(--surface-2); color: var(--muted);
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
.badge::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: currentColor; }
|
||||||
|
.badge.plain::before { display: none; }
|
||||||
|
.badge.st-triggered, .badge.st-firing { background: var(--crit-soft); color: var(--crit); }
|
||||||
|
.badge.st-acknowledged { background: var(--warn-soft); color: var(--warn); }
|
||||||
|
.badge.st-snoozed { background: var(--snooze-soft); color: var(--snooze); }
|
||||||
|
.badge.st-resolved { background: var(--ok-soft); color: var(--ok); }
|
||||||
|
.badge.sev-critical { background: var(--crit-soft); color: var(--crit); }
|
||||||
|
.badge.sev-warning { background: var(--warn-soft); color: var(--warn); }
|
||||||
|
.badge.sev-info { background: var(--info-soft); color: var(--info); }
|
||||||
|
|
||||||
|
/* ---------- incident detail ---------- */
|
||||||
|
|
||||||
|
.detail { padding: 0 16px calc(96px + var(--safe-bottom)); }
|
||||||
|
.detail-head {
|
||||||
|
position: sticky; top: 0; z-index: 5;
|
||||||
|
display: flex; align-items: center; gap: 4px;
|
||||||
|
height: calc(var(--topbar-h) + var(--safe-top));
|
||||||
|
margin: 0 -16px; padding: var(--safe-top) 8px 0;
|
||||||
|
background: color-mix(in srgb, var(--bg) 88%, transparent);
|
||||||
|
backdrop-filter: saturate(1.4) blur(12px);
|
||||||
|
-webkit-backdrop-filter: saturate(1.4) blur(12px);
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
.detail-head .crumb { font-weight: 600; color: var(--muted); font-size: 14px; }
|
||||||
|
.detail-title { font-size: 21px; font-weight: 750; letter-spacing: -0.01em; margin: 16px 0 8px; overflow-wrap: anywhere; }
|
||||||
|
.detail-badges { display: flex; flex-wrap: wrap; gap: 6px; margin-bottom: 14px; }
|
||||||
|
|
||||||
|
.card {
|
||||||
|
background: var(--surface);
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
box-shadow: var(--shadow);
|
||||||
|
}
|
||||||
|
.card + .card, .section + .section { margin-top: 14px; }
|
||||||
|
.card-pad { padding: 14px; }
|
||||||
|
|
||||||
|
.facts { display: grid; grid-template-columns: auto 1fr; gap: 8px 16px; margin: 0; padding: 14px; font-size: 14px; }
|
||||||
|
.facts dt { color: var(--muted); }
|
||||||
|
.facts dd { margin: 0; overflow-wrap: anywhere; }
|
||||||
|
.facts .sub { color: var(--faint); }
|
||||||
|
|
||||||
|
.section { margin-top: 22px; }
|
||||||
|
.section-title {
|
||||||
|
display: flex; align-items: baseline; justify-content: space-between; gap: 8px;
|
||||||
|
font-size: 13px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.06em;
|
||||||
|
color: var(--muted); margin: 0 2px 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.labels-wrap { display: flex; flex-wrap: wrap; gap: 6px; }
|
||||||
|
.label {
|
||||||
|
display: inline-flex; max-width: 100%;
|
||||||
|
font-family: var(--mono); font-size: 12px;
|
||||||
|
border: 1px solid var(--border); border-radius: var(--radius-sm);
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
.label > span { padding: 2px 6px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
.label > span:first-child { background: var(--surface-2); color: var(--muted); }
|
||||||
|
|
||||||
|
.alert-item { padding: 12px 14px; display: grid; gap: 6px; }
|
||||||
|
.alert-item + .alert-item { border-top: 1px solid var(--border); }
|
||||||
|
.alert-item-head { display: flex; align-items: center; justify-content: space-between; gap: 8px; }
|
||||||
|
.alert-item-name { font-weight: 650; overflow-wrap: anywhere; }
|
||||||
|
.alert-item-summary { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; }
|
||||||
|
.alert-item-foot { display: flex; flex-wrap: wrap; gap: 4px 12px; font-size: 13px; color: var(--faint); }
|
||||||
|
.alert-item-foot a { color: var(--accent); font-weight: 600; }
|
||||||
|
details > summary { cursor: pointer; color: var(--muted); font-size: 13px; font-weight: 600; list-style: none; }
|
||||||
|
details > summary::-webkit-details-marker { display: none; }
|
||||||
|
details > summary::before { content: "▸ "; }
|
||||||
|
details[open] > summary::before { content: "▾ "; }
|
||||||
|
details[open] > summary { margin-bottom: 8px; }
|
||||||
|
|
||||||
|
.timeline { list-style: none; margin: 0; padding: 4px 0; }
|
||||||
|
.tl-item {
|
||||||
|
position: relative;
|
||||||
|
display: grid; grid-template-columns: 20px 1fr; gap: 10px;
|
||||||
|
padding: 8px 14px;
|
||||||
|
}
|
||||||
|
.tl-item::before {
|
||||||
|
content: ""; position: absolute; left: 23px; top: 0; bottom: 0; width: 2px; background: var(--border);
|
||||||
|
}
|
||||||
|
.tl-item:first-child::before { top: 16px; }
|
||||||
|
.tl-item:last-child::before { bottom: calc(100% - 16px); }
|
||||||
|
.tl-dot {
|
||||||
|
position: relative; z-index: 1;
|
||||||
|
width: 10px; height: 10px; margin: 5px 0 0 5px; border-radius: 50%;
|
||||||
|
background: var(--surface); border: 2px solid var(--faint);
|
||||||
|
}
|
||||||
|
.tl-triggered .tl-dot, .tl-notify_failed .tl-dot, .tl-deadman_silent .tl-dot { border-color: var(--crit); background: var(--crit); }
|
||||||
|
.tl-acknowledged .tl-dot { border-color: var(--warn); background: var(--warn); }
|
||||||
|
.tl-resolved .tl-dot { border-color: var(--ok); background: var(--ok); }
|
||||||
|
.tl-snoozed .tl-dot { border-color: var(--snooze); }
|
||||||
|
.tl-note .tl-dot { border-color: var(--accent); background: var(--accent); }
|
||||||
|
.tl-body { min-width: 0; font-size: 14px; }
|
||||||
|
.tl-text { overflow-wrap: anywhere; }
|
||||||
|
.tl-text .who { font-weight: 650; }
|
||||||
|
.tl-time { color: var(--faint); font-size: 12px; }
|
||||||
|
.tl-note .note {
|
||||||
|
margin-top: 6px; padding: 10px 12px;
|
||||||
|
background: var(--surface-2); border-radius: var(--radius-sm);
|
||||||
|
white-space: pre-wrap; overflow-wrap: anywhere;
|
||||||
|
}
|
||||||
|
.note-actions { display: flex; justify-content: flex-end; }
|
||||||
|
.note-actions .btn { color: var(--muted); }
|
||||||
|
|
||||||
|
/* The action bar sits at the bottom of the screen on a phone, and at the
|
||||||
|
bottom of the detail pane on desktop. */
|
||||||
|
.actionbar {
|
||||||
|
position: fixed; left: 0; right: 0; bottom: 0; z-index: 15;
|
||||||
|
display: flex; gap: 8px;
|
||||||
|
padding: 10px 16px calc(10px + var(--safe-bottom));
|
||||||
|
background: var(--surface);
|
||||||
|
border-top: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
.actionbar .btn { min-height: 48px; }
|
||||||
|
.actionbar .btn-primary { flex: 1; font-size: 16px; }
|
||||||
|
|
||||||
|
.detail-placeholder {
|
||||||
|
display: grid; place-items: center; height: 100%;
|
||||||
|
color: var(--faint); text-align: center; padding: 32px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---------- sheet (dialog) ---------- */
|
||||||
|
|
||||||
|
.sheet {
|
||||||
|
width: 100%; max-width: 100%;
|
||||||
|
max-height: 88dvh;
|
||||||
|
margin: auto 0 0; padding: 0;
|
||||||
|
border: 0; border-radius: 16px 16px 0 0;
|
||||||
|
background: var(--surface); color: var(--text);
|
||||||
|
box-shadow: var(--shadow-lg);
|
||||||
|
overflow: auto;
|
||||||
|
}
|
||||||
|
.sheet::backdrop { background: rgb(0 0 0 / 40%); }
|
||||||
|
.sheet[open] { animation: sheet-up 0.18s ease-out; }
|
||||||
|
@keyframes sheet-up { from { transform: translateY(24px); opacity: 0.6; } }
|
||||||
|
.sheet-inner { padding: 8px 16px calc(16px + var(--safe-bottom)); }
|
||||||
|
.sheet-grab { width: 40px; height: 4px; margin: 0 auto 12px; border-radius: 2px; background: var(--border-strong); }
|
||||||
|
.sheet-title { font-size: 17px; font-weight: 700; margin: 0 0 4px; }
|
||||||
|
.sheet-text { color: var(--muted); margin: 0 0 14px; font-size: 14px; }
|
||||||
|
.sheet-form { display: grid; gap: 12px; }
|
||||||
|
.sheet-actions { display: flex; gap: 8px; margin-top: 16px; }
|
||||||
|
.sheet-actions .btn { flex: 1; }
|
||||||
|
|
||||||
|
.menu { list-style: none; margin: 0 -4px; padding: 0; }
|
||||||
|
.menu-item {
|
||||||
|
display: flex; align-items: center; gap: 12px;
|
||||||
|
width: 100%; min-height: 50px; padding: 0 12px;
|
||||||
|
background: none; border: 0; border-radius: var(--radius-sm);
|
||||||
|
text-align: left; font-size: 16px; cursor: pointer;
|
||||||
|
}
|
||||||
|
.menu-item:hover { background: var(--surface-2); }
|
||||||
|
.menu-item .icon { color: var(--muted); flex: none; }
|
||||||
|
.menu-item .menu-sub { margin-left: auto; color: var(--faint); font-size: 13px; }
|
||||||
|
.menu-item.danger, .menu-item.danger .icon { color: var(--crit); }
|
||||||
|
.menu-item[aria-checked="true"] { font-weight: 700; }
|
||||||
|
.menu-item[aria-checked="true"]::after { content: "✓"; margin-left: 8px; color: var(--accent); }
|
||||||
|
.menu-sep { height: 1px; background: var(--border); margin: 6px 12px; }
|
||||||
|
|
||||||
|
/* ---------- toast ---------- */
|
||||||
|
|
||||||
|
.toast {
|
||||||
|
position: fixed; left: 50%; z-index: 50;
|
||||||
|
bottom: calc(var(--tabbar-h) + var(--safe-bottom) + 12px);
|
||||||
|
transform: translateX(-50%);
|
||||||
|
max-width: calc(100% - 32px);
|
||||||
|
padding: 10px 16px; border-radius: var(--radius);
|
||||||
|
background: var(--text); color: var(--bg);
|
||||||
|
font-size: 14px; font-weight: 600;
|
||||||
|
box-shadow: var(--shadow-lg);
|
||||||
|
}
|
||||||
|
.toast.error { background: var(--crit); color: #fff; }
|
||||||
|
.app.detail-open ~ .toast { bottom: calc(80px + var(--safe-bottom)); }
|
||||||
|
|
||||||
|
/* ---------- on-call ---------- */
|
||||||
|
|
||||||
|
.page-head { display: flex; align-items: center; justify-content: space-between; gap: 8px; margin: 16px auto 12px; }
|
||||||
|
.page-head h2 { font-size: 13px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.06em; color: var(--muted); }
|
||||||
|
|
||||||
|
.now-card { display: flex; align-items: center; gap: 14px; padding: 16px; margin-top: 16px; }
|
||||||
|
.avatar {
|
||||||
|
flex: none; display: grid; place-items: center;
|
||||||
|
width: 44px; height: 44px; border-radius: 50%;
|
||||||
|
background: var(--accent-soft); color: var(--accent);
|
||||||
|
font-weight: 750; font-size: 17px; text-transform: uppercase;
|
||||||
|
}
|
||||||
|
.avatar.none { background: var(--surface-2); color: var(--faint); }
|
||||||
|
.now-label { color: var(--muted); font-size: 13px; font-weight: 600; }
|
||||||
|
.now-name { font-size: 20px; font-weight: 750; }
|
||||||
|
.you { color: var(--accent); font-weight: 650; font-size: 13px; margin-left: 6px; }
|
||||||
|
|
||||||
|
.week-nav { display: flex; align-items: center; gap: 4px; }
|
||||||
|
.week-nav .label { font-size: 14px; font-weight: 650; min-width: 9em; text-align: center; }
|
||||||
|
.days { list-style: none; margin: 0; padding: 0; }
|
||||||
|
.day { display: grid; grid-template-columns: 3.2em 4.2em 1fr; align-items: center; gap: 8px; min-height: 50px; padding: 0 14px; }
|
||||||
|
.day + .day { border-top: 1px solid var(--border); }
|
||||||
|
.day-name { font-weight: 650; }
|
||||||
|
.day-date { color: var(--faint); font-size: 13px; }
|
||||||
|
.day-who { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||||
|
.day-who.nobody { color: var(--faint); font-style: italic; }
|
||||||
|
.day.today { background: var(--accent-soft); }
|
||||||
|
.day.today:first-child { border-radius: var(--radius) var(--radius) 0 0; }
|
||||||
|
.day.today:last-child { border-radius: 0 0 var(--radius) var(--radius); }
|
||||||
|
.day.today .day-name { color: var(--accent); }
|
||||||
|
.day.past { opacity: 0.6; }
|
||||||
|
|
||||||
|
.shift-list { list-style: none; margin: 0; padding: 0; }
|
||||||
|
.shift-list li { display: flex; justify-content: space-between; padding: 12px 14px; }
|
||||||
|
.shift-list li + li { border-top: 1px solid var(--border); }
|
||||||
|
.shift-list .muted { color: var(--faint); }
|
||||||
|
|
||||||
|
/* ---------- alerts page ---------- */
|
||||||
|
|
||||||
|
.view-alerts .chips { padding-left: 0; padding-right: 0; }
|
||||||
|
.view-alerts .list { padding-left: 0; padding-right: 0; }
|
||||||
|
.row.st-firing { --sev: var(--crit); }
|
||||||
|
.row.st-resolved { --sev: var(--ok); }
|
||||||
|
.row.no-link { cursor: default; }
|
||||||
|
|
||||||
|
/* ---------- account ---------- */
|
||||||
|
|
||||||
|
.account-card { display: flex; align-items: center; gap: 14px; padding: 16px; margin-top: 16px; }
|
||||||
|
.account-name { font-size: 18px; font-weight: 750; }
|
||||||
|
.account-email { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; }
|
||||||
|
.pw-form { display: grid; gap: 12px; padding: 16px; }
|
||||||
|
.form-ok {
|
||||||
|
margin: 0; padding: 10px 12px;
|
||||||
|
background: var(--ok-soft); color: var(--ok);
|
||||||
|
border-radius: var(--radius-sm); font-size: 14px;
|
||||||
|
}
|
||||||
|
.kbd-table { width: 100%; border-collapse: collapse; font-size: 14px; }
|
||||||
|
.kbd-table td { padding: 8px 14px; border-top: 1px solid var(--border); }
|
||||||
|
.kbd-table tr:first-child td { border-top: 0; }
|
||||||
|
kbd {
|
||||||
|
display: inline-block; min-width: 1.6em; padding: 1px 6px;
|
||||||
|
font-family: var(--mono); font-size: 12px; text-align: center;
|
||||||
|
background: var(--surface-2); border: 1px solid var(--border-strong); border-bottom-width: 2px;
|
||||||
|
border-radius: 4px;
|
||||||
|
}
|
||||||
|
.only-desktop { display: none; }
|
||||||
|
.foot-note { color: var(--faint); font-size: 13px; text-align: center; margin: 24px auto; }
|
||||||
|
|
||||||
|
/* ---------- desktop ---------- */
|
||||||
|
|
||||||
|
@media (min-width: 900px) {
|
||||||
|
:root { --tabbar-h: 0px; }
|
||||||
|
|
||||||
|
.app { display: grid; grid-template-columns: 220px 1fr; height: 100dvh; }
|
||||||
|
|
||||||
|
.nav {
|
||||||
|
position: static; grid-row: 1 / span 2;
|
||||||
|
display: flex; flex-direction: column; gap: 2px;
|
||||||
|
height: auto; padding: 16px 12px;
|
||||||
|
background: var(--surface);
|
||||||
|
border-top: 0; border-right: 1px solid var(--border);
|
||||||
|
backdrop-filter: none;
|
||||||
|
}
|
||||||
|
.nav-brand {
|
||||||
|
display: flex; align-items: center; gap: 10px;
|
||||||
|
padding: 4px 10px 18px; font-size: 18px; font-weight: 750; letter-spacing: -0.01em;
|
||||||
|
}
|
||||||
|
.nav-link {
|
||||||
|
flex-direction: row; justify-content: flex-start; gap: 12px;
|
||||||
|
min-height: 40px; padding: 0 10px; border-radius: var(--radius-sm);
|
||||||
|
color: var(--muted); font-size: 14px;
|
||||||
|
}
|
||||||
|
.nav-link:hover { background: var(--surface-2); }
|
||||||
|
.nav-link[aria-current="page"] { background: var(--accent-soft); color: var(--accent); }
|
||||||
|
.nav-link svg { width: 20px; height: 20px; }
|
||||||
|
.nav-badge { position: static; margin-left: auto; }
|
||||||
|
|
||||||
|
.topbar { display: none; }
|
||||||
|
.view { padding-bottom: 0; overflow: auto; height: 100dvh; }
|
||||||
|
.view-page { padding: 8px 32px 32px; }
|
||||||
|
|
||||||
|
.view-queue {
|
||||||
|
display: grid; grid-template-columns: minmax(340px, 420px) 1fr;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
.view-queue .pane { overflow: auto; height: 100dvh; }
|
||||||
|
.pane-list { border-right: 1px solid var(--border); }
|
||||||
|
.pane-list .chips { position: sticky; top: 0; z-index: 2; background: var(--bg); padding-top: 16px; }
|
||||||
|
.view-queue:not(.has-detail) .pane-detail { display: block; }
|
||||||
|
|
||||||
|
/* On desktop the list stays visible next to the detail. */
|
||||||
|
.app.detail-open .nav { display: flex; }
|
||||||
|
.app.detail-open .pane-list { display: block; }
|
||||||
|
.detail-head .back { display: none; }
|
||||||
|
.detail-head { padding-left: 16px; }
|
||||||
|
|
||||||
|
.pane-detail { position: relative; display: flex; flex-direction: column; }
|
||||||
|
.detail { flex: 1; padding: 0 32px 24px; max-width: 900px; width: 100%; }
|
||||||
|
.detail-head { margin: 0 -32px; padding-left: 32px; }
|
||||||
|
.actionbar {
|
||||||
|
position: sticky; bottom: 0;
|
||||||
|
padding: 12px 32px;
|
||||||
|
}
|
||||||
|
.actionbar .btn-primary { flex: 0 1 240px; }
|
||||||
|
|
||||||
|
.sheet {
|
||||||
|
width: min(440px, calc(100% - 32px));
|
||||||
|
margin: auto; border-radius: 14px;
|
||||||
|
}
|
||||||
|
.sheet-grab { display: none; }
|
||||||
|
.sheet-inner { padding: 20px; }
|
||||||
|
|
||||||
|
.toast, .app.detail-open ~ .toast { bottom: 24px; }
|
||||||
|
.only-desktop { display: block; }
|
||||||
|
}
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 2.0 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
@@ -0,0 +1,4 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64">
|
||||||
|
<rect width="64" height="64" rx="14" fill="#1b1e25"/>
|
||||||
|
<path d="M10 34h11l5-12 8 22 6-15 3 5h11" fill="none" stroke="#ff6b61" stroke-width="4.5" stroke-linecap="round" stroke-linejoin="round"/>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 265 B |
@@ -0,0 +1,89 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
|
||||||
|
<meta name="color-scheme" content="light dark">
|
||||||
|
<meta name="theme-color" content="#f5f6f8" media="(prefers-color-scheme: light)">
|
||||||
|
<meta name="theme-color" content="#0f1115" media="(prefers-color-scheme: dark)">
|
||||||
|
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||||
|
<meta name="apple-mobile-web-app-status-bar-style" content="default">
|
||||||
|
<meta name="apple-mobile-web-app-title" content="terdut">
|
||||||
|
<title>terdut</title>
|
||||||
|
<link rel="manifest" href="/manifest.webmanifest">
|
||||||
|
<link rel="icon" href="/icon.svg" type="image/svg+xml">
|
||||||
|
<link rel="apple-touch-icon" href="/apple-touch-icon.png">
|
||||||
|
<link rel="stylesheet" href="/app.css">
|
||||||
|
<script type="module" src="/js/app.js"></script>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div id="boot" class="boot" aria-busy="true"><span class="spinner"></span></div>
|
||||||
|
|
||||||
|
<main id="login" class="login" hidden>
|
||||||
|
<form id="login-form" class="login-card" autocomplete="on">
|
||||||
|
<div class="login-brand">
|
||||||
|
<img src="/icon.svg" alt="" width="40" height="40">
|
||||||
|
<h1>terdut</h1>
|
||||||
|
</div>
|
||||||
|
<label>
|
||||||
|
<span>Username</span>
|
||||||
|
<input name="username" autocomplete="username" autocapitalize="none" spellcheck="false" required>
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
<span>Password</span>
|
||||||
|
<input name="password" type="password" autocomplete="current-password" required>
|
||||||
|
</label>
|
||||||
|
<p class="form-error" role="alert" hidden></p>
|
||||||
|
<button class="btn btn-primary btn-block" type="submit">Sign in</button>
|
||||||
|
<p class="login-hint">No password yet? Ask an admin to set one, or run
|
||||||
|
<code>PUT /api/users/{id}/password</code> with your API key.</p>
|
||||||
|
</form>
|
||||||
|
</main>
|
||||||
|
|
||||||
|
<div id="app" class="app" hidden>
|
||||||
|
<nav class="nav" aria-label="Sections">
|
||||||
|
<a class="nav-brand" href="/">
|
||||||
|
<img src="/icon.svg" alt="" width="28" height="28">
|
||||||
|
<span>terdut</span>
|
||||||
|
</a>
|
||||||
|
<a class="nav-link" href="/" data-section="queue">
|
||||||
|
<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M4 6h16M4 12h16M4 18h10"/></svg>
|
||||||
|
<span class="nav-label">Queue</span>
|
||||||
|
<span class="nav-badge" data-badge hidden></span>
|
||||||
|
</a>
|
||||||
|
<a class="nav-link" href="/oncall" data-section="oncall">
|
||||||
|
<svg viewBox="0 0 24 24" aria-hidden="true"><rect x="3.5" y="5" width="17" height="15" rx="2"/><path d="M3.5 10h17M8 3v4M16 3v4"/></svg>
|
||||||
|
<span class="nav-label">On-call</span>
|
||||||
|
</a>
|
||||||
|
<a class="nav-link" href="/alerts" data-section="alerts">
|
||||||
|
<svg viewBox="0 0 24 24" aria-hidden="true"><path d="M6 16V11a6 6 0 0 1 12 0v5l1.5 2h-15z"/><path d="M10 20.5a2 2 0 0 0 4 0"/></svg>
|
||||||
|
<span class="nav-label">Alerts</span>
|
||||||
|
</a>
|
||||||
|
<a class="nav-link" href="/more" data-section="more">
|
||||||
|
<svg viewBox="0 0 24 24" aria-hidden="true"><circle cx="12" cy="8" r="3.5"/><path d="M5 20a7 7 0 0 1 14 0"/></svg>
|
||||||
|
<span class="nav-label">Account</span>
|
||||||
|
</a>
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
<header class="topbar">
|
||||||
|
<h1 class="topbar-title" id="topbar-title">Queue</h1>
|
||||||
|
<span class="open-pill" id="open-pill" hidden></span>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<section id="view-queue" class="view view-queue" data-view="queue">
|
||||||
|
<div class="pane pane-list">
|
||||||
|
<div class="chips" id="queue-filters" role="tablist" aria-label="Filter"></div>
|
||||||
|
<div id="queue-list" class="list"></div>
|
||||||
|
</div>
|
||||||
|
<div class="pane pane-detail" id="detail" aria-live="polite"></div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section id="view-oncall" class="view view-page" data-view="oncall" hidden></section>
|
||||||
|
<section id="view-alerts" class="view view-page" data-view="alerts" hidden></section>
|
||||||
|
<section id="view-more" class="view view-page" data-view="more" hidden></section>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<dialog id="sheet" class="sheet"></dialog>
|
||||||
|
<div id="toast" class="toast" role="status" aria-live="polite" hidden></div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
// Account: who you are signed in as, changing your password, signing out.
|
||||||
|
|
||||||
|
import * as api from './api.js';
|
||||||
|
import { h, clear, icon, toast } from './ui.js';
|
||||||
|
import { initial } from './format.js';
|
||||||
|
import { state } from './state.js';
|
||||||
|
import { signOut } from './app.js';
|
||||||
|
|
||||||
|
const view = () => document.getElementById('view-more');
|
||||||
|
|
||||||
|
// Rendered once per visit rather than on every poll, so a half-typed password
|
||||||
|
// is never wiped out from under you.
|
||||||
|
export function show() {
|
||||||
|
render();
|
||||||
|
}
|
||||||
|
|
||||||
|
function render() {
|
||||||
|
const { user, has_password: hasPassword } = state.me;
|
||||||
|
clear(view(),
|
||||||
|
h('div', { class: 'card account-card' },
|
||||||
|
h('div', { class: 'avatar', text: initial(user.username) }),
|
||||||
|
h('div', {},
|
||||||
|
h('div', { class: 'account-name', text: user.username }),
|
||||||
|
h('div', { class: 'account-email', text: user.email }))),
|
||||||
|
|
||||||
|
h('div', { class: 'page-head' }, h('h2', { text: hasPassword ? 'Change password' : 'Set a password' })),
|
||||||
|
passwordForm(user, hasPassword),
|
||||||
|
|
||||||
|
h('div', { class: 'only-desktop' },
|
||||||
|
h('div', { class: 'page-head' }, h('h2', { text: 'Keyboard' })),
|
||||||
|
h('div', { class: 'card' }, shortcuts())),
|
||||||
|
|
||||||
|
h('div', { class: 'page-head' }),
|
||||||
|
h('button', { class: 'btn btn-block', type: 'button', onclick: signOut }, icon('logout'), 'Sign out'),
|
||||||
|
h('p', { class: 'foot-note', text: 'Schedule editing, statistics and user management are in terdut-tui for now.' }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function passwordForm(user, hasPassword) {
|
||||||
|
const err = h('p', { class: 'form-error', role: 'alert', hidden: true });
|
||||||
|
const ok = h('p', { class: 'form-ok', role: 'status', hidden: true });
|
||||||
|
const current = hasPassword
|
||||||
|
? h('input', { name: 'current', type: 'password', autocomplete: 'current-password', required: true })
|
||||||
|
: null;
|
||||||
|
const next = h('input', { name: 'next', type: 'password', autocomplete: 'new-password', required: true, minlength: '10' });
|
||||||
|
const again = h('input', { name: 'again', type: 'password', autocomplete: 'new-password', required: true, minlength: '10' });
|
||||||
|
const submit = h('button', { class: 'btn btn-primary', type: 'submit', text: 'Save password' });
|
||||||
|
|
||||||
|
// A hidden username field lets password managers file the new password
|
||||||
|
// under the right account.
|
||||||
|
const form = h('form', { class: 'card pw-form', autocomplete: 'on' },
|
||||||
|
h('input', { type: 'text', name: 'username', autocomplete: 'username', value: user.username, hidden: true, readonly: true }),
|
||||||
|
current && h('label', {}, h('span', { text: 'Current password' }), current),
|
||||||
|
h('label', {}, h('span', { text: 'New password' }), next),
|
||||||
|
h('label', {}, h('span', { text: 'Repeat new password' }), again),
|
||||||
|
err, ok, submit,
|
||||||
|
);
|
||||||
|
|
||||||
|
form.addEventListener('submit', async (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
err.hidden = true;
|
||||||
|
ok.hidden = true;
|
||||||
|
if (next.value !== again.value) {
|
||||||
|
err.textContent = 'The new passwords do not match.';
|
||||||
|
err.hidden = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
submit.disabled = true;
|
||||||
|
try {
|
||||||
|
await api.setPassword(user.id, next.value, current ? current.value : '');
|
||||||
|
state.me.has_password = true;
|
||||||
|
form.reset();
|
||||||
|
if (!current) {
|
||||||
|
// From now on the form needs the current-password field.
|
||||||
|
render();
|
||||||
|
toast('Password saved');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ok.textContent = 'Password saved. Other devices have been signed out.';
|
||||||
|
ok.hidden = false;
|
||||||
|
} catch (ex) {
|
||||||
|
err.textContent = ex.message;
|
||||||
|
err.hidden = false;
|
||||||
|
} finally {
|
||||||
|
submit.disabled = false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return form;
|
||||||
|
}
|
||||||
|
|
||||||
|
function shortcuts() {
|
||||||
|
const rows = [
|
||||||
|
['j / k', 'Move through the queue'],
|
||||||
|
['Enter', 'Open incident'],
|
||||||
|
['Esc', 'Back to the queue'],
|
||||||
|
['f', 'Cycle the queue filter'],
|
||||||
|
['a / A', 'Acknowledge / clear acknowledgement'],
|
||||||
|
['R', 'Resolve (asks first)'],
|
||||||
|
['s', 'Assign'],
|
||||||
|
['z / Z', 'Snooze / end snooze'],
|
||||||
|
['c', 'Add a note'],
|
||||||
|
['x', 'Archive / unarchive a resolved incident'],
|
||||||
|
['r', 'Refresh now'],
|
||||||
|
];
|
||||||
|
return h('table', { class: 'kbd-table' },
|
||||||
|
h('tbody', {}, rows.map(([k, v]) =>
|
||||||
|
h('tr', {},
|
||||||
|
h('td', {}, k.split(' / ').map((x, i) => [i ? ' / ' : '', h('kbd', { text: x })])),
|
||||||
|
h('td', { text: v })))));
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
// The alert feed: Alertmanager's own records, read-only. Each row leads to the
|
||||||
|
// incident it belongs to, which is where anything can be done about it.
|
||||||
|
|
||||||
|
import * as api from './api.js';
|
||||||
|
import { h, clear, badge, emptyState, spinner } from './ui.js';
|
||||||
|
import { age, severityClass, labelSummary } from './format.js';
|
||||||
|
|
||||||
|
const FILTERS = [
|
||||||
|
{ id: 'firing', label: 'Firing', query: { status: 'firing' } },
|
||||||
|
{ id: 'resolved', label: 'Resolved', query: { status: 'resolved' } },
|
||||||
|
{ id: 'all', label: 'All', query: {} },
|
||||||
|
{ id: 'archived', label: 'Archived', query: { archived: 'true' } },
|
||||||
|
];
|
||||||
|
|
||||||
|
const view = () => document.getElementById('view-alerts');
|
||||||
|
|
||||||
|
let filter = 'firing';
|
||||||
|
let items = null;
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
export function show() {
|
||||||
|
render();
|
||||||
|
refresh();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function refresh() {
|
||||||
|
const requested = filter;
|
||||||
|
const f = FILTERS.find((x) => x.id === filter);
|
||||||
|
try {
|
||||||
|
const result = await api.alerts({ ...f.query, limit: 200 });
|
||||||
|
if (requested !== filter) return;
|
||||||
|
items = result;
|
||||||
|
error = null;
|
||||||
|
} catch (err) {
|
||||||
|
if (requested !== filter) return;
|
||||||
|
error = err.message;
|
||||||
|
}
|
||||||
|
render();
|
||||||
|
}
|
||||||
|
|
||||||
|
function setFilter(id) {
|
||||||
|
if (id === filter) return;
|
||||||
|
filter = id;
|
||||||
|
items = null;
|
||||||
|
render();
|
||||||
|
refresh();
|
||||||
|
}
|
||||||
|
|
||||||
|
function render() {
|
||||||
|
const chips = h('div', { class: 'chips', role: 'tablist', 'aria-label': 'Filter' },
|
||||||
|
FILTERS.map((f) => h('button', {
|
||||||
|
class: 'chip',
|
||||||
|
type: 'button',
|
||||||
|
role: 'tab',
|
||||||
|
'aria-selected': String(f.id === filter),
|
||||||
|
onclick: () => setFilter(f.id),
|
||||||
|
text: f.label,
|
||||||
|
})));
|
||||||
|
|
||||||
|
let body;
|
||||||
|
if (error && !items) body = h('div', { class: 'load-error', text: error });
|
||||||
|
else if (!items) body = spinner();
|
||||||
|
else if (!items.length) body = emptyState(filter === 'firing' ? 'Nothing firing' : 'No alerts', '', filter === 'firing' ? 'checkCircle' : null);
|
||||||
|
else {
|
||||||
|
body = h('div', { class: 'list' },
|
||||||
|
error && h('div', { class: 'load-error', text: `Showing older data: ${error}` }),
|
||||||
|
items.map(row));
|
||||||
|
}
|
||||||
|
clear(view(), h('div', {}, chips, body));
|
||||||
|
}
|
||||||
|
|
||||||
|
function row(a) {
|
||||||
|
const summary = (a.annotations && a.annotations.summary) || '';
|
||||||
|
const sev = a.labels && a.labels.severity;
|
||||||
|
const labels = labelSummary(Object.fromEntries(
|
||||||
|
Object.entries(a.labels || {}).filter(([k]) => k !== 'severity')));
|
||||||
|
const linked = a.incident_id != null;
|
||||||
|
return h(linked ? 'a' : 'div', {
|
||||||
|
class: `row st-${a.status} ${linked ? '' : 'no-link'}`,
|
||||||
|
href: linked ? `/incidents/${a.incident_id}` : null,
|
||||||
|
},
|
||||||
|
h('div', { class: 'row-title', text: a.name }),
|
||||||
|
h('div', { class: 'row-age', title: a.starts_at, text: age(a.status === 'firing' ? a.starts_at : a.received_at) }),
|
||||||
|
h('div', { class: 'row-meta' },
|
||||||
|
badge(a.status === 'firing' ? 'Firing' : 'Resolved', `st-${a.status}`),
|
||||||
|
sev && badge(sev, `plain ${severityClass(sev)}`),
|
||||||
|
summary && h('span', { text: summary }),
|
||||||
|
labels && h('span', { class: 'labels', text: labels }),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
// The terdut-server client. The page is served by the server itself, so every
|
||||||
|
// call is same-origin and carries the session cookie.
|
||||||
|
|
||||||
|
export class ApiError extends Error {
|
||||||
|
constructor(status, message) {
|
||||||
|
super(message);
|
||||||
|
this.name = 'ApiError';
|
||||||
|
this.status = status;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Called whenever the server says the session is gone, so the app can put the
|
||||||
|
// login form back up wherever the user happened to be.
|
||||||
|
let onUnauthorized = () => {};
|
||||||
|
export function setUnauthorizedHandler(fn) {
|
||||||
|
onUnauthorized = fn;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function call(method, path, { query, body, signal } = {}) {
|
||||||
|
const url = new URL('/api' + path, location.origin);
|
||||||
|
for (const [k, v] of Object.entries(query || {})) {
|
||||||
|
if (v === '' || v == null) continue;
|
||||||
|
url.searchParams.set(k, v);
|
||||||
|
}
|
||||||
|
|
||||||
|
const headers = { Accept: 'application/json' };
|
||||||
|
if (body !== undefined) headers['Content-Type'] = 'application/json';
|
||||||
|
|
||||||
|
let resp;
|
||||||
|
try {
|
||||||
|
resp = await fetch(url, {
|
||||||
|
method,
|
||||||
|
headers,
|
||||||
|
body: body === undefined ? undefined : JSON.stringify(body),
|
||||||
|
credentials: 'same-origin',
|
||||||
|
signal,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
if (err.name === 'AbortError') throw err;
|
||||||
|
throw new ApiError(0, 'Cannot reach the server.');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (resp.status === 204) return null;
|
||||||
|
|
||||||
|
let data = null;
|
||||||
|
try {
|
||||||
|
data = await resp.json();
|
||||||
|
} catch {
|
||||||
|
/* non-JSON body: keep null */
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!resp.ok) {
|
||||||
|
if (resp.status === 401 && path !== '/login') onUnauthorized();
|
||||||
|
const message = (data && data.error) || `Server answered ${resp.status}.`;
|
||||||
|
throw new ApiError(resp.status, message);
|
||||||
|
}
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
// session
|
||||||
|
export const me = () => call('GET', '/me');
|
||||||
|
export const login = (username, password) => call('POST', '/login', { body: { username, password } });
|
||||||
|
export const logout = () => call('POST', '/logout');
|
||||||
|
export const setPassword = (userID, password, currentPassword) =>
|
||||||
|
call('PUT', `/users/${userID}/password`, { body: { password, current_password: currentPassword } });
|
||||||
|
|
||||||
|
// users
|
||||||
|
export const users = () => call('GET', '/users');
|
||||||
|
|
||||||
|
// incidents
|
||||||
|
export const incidents = (query, opts) => call('GET', '/incidents', { query, ...opts });
|
||||||
|
export const incident = (id) => call('GET', `/incidents/${id}`);
|
||||||
|
export const timeline = (id) => call('GET', `/incidents/${id}/timeline`);
|
||||||
|
|
||||||
|
export const acknowledge = (id) => call('POST', `/incidents/${id}/acknowledge`);
|
||||||
|
export const unacknowledge = (id) => call('DELETE', `/incidents/${id}/acknowledge`);
|
||||||
|
export const resolve = (id) => call('POST', `/incidents/${id}/resolve`);
|
||||||
|
export const assign = (id, userID) => call('POST', `/incidents/${id}/assign`, { body: { user_id: userID } });
|
||||||
|
export const snooze = (id, spec) => call('POST', `/incidents/${id}/snooze`, { body: spec });
|
||||||
|
export const unsnooze = (id) => call('DELETE', `/incidents/${id}/snooze`);
|
||||||
|
export const archive = (id) => call('POST', `/incidents/${id}/archive`);
|
||||||
|
export const unarchive = (id) => call('DELETE', `/incidents/${id}/archive`);
|
||||||
|
export const addNote = (id, content) => call('POST', `/incidents/${id}/notes`, { body: { content } });
|
||||||
|
export const deleteNote = (id, eventID) => call('DELETE', `/incidents/${id}/notes/${eventID}`);
|
||||||
|
|
||||||
|
// alerts
|
||||||
|
export const alerts = (query, opts) => call('GET', '/alerts', { query, ...opts });
|
||||||
|
|
||||||
|
// schedule
|
||||||
|
export const schedule = (from, to) => call('GET', '/schedule', { query: { from, to } });
|
||||||
|
export async function onCallNow() {
|
||||||
|
try {
|
||||||
|
return await call('GET', '/schedule/current');
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof ApiError && err.status === 404) return null;
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,234 @@
|
|||||||
|
// Entry point: session, routing, badges and keyboard.
|
||||||
|
|
||||||
|
import * as api from './api.js';
|
||||||
|
import * as ui from './ui.js';
|
||||||
|
import * as poll from './poll.js';
|
||||||
|
import { state, reset } from './state.js';
|
||||||
|
import * as queue from './queue.js';
|
||||||
|
import * as incident from './incident.js';
|
||||||
|
import * as oncall from './oncall.js';
|
||||||
|
import * as alerts from './alerts.js';
|
||||||
|
import * as account from './account.js';
|
||||||
|
|
||||||
|
const $ = (id) => document.getElementById(id);
|
||||||
|
|
||||||
|
// One route per section; /incidents/{id} is the queue with a detail open.
|
||||||
|
const SECTIONS = {
|
||||||
|
queue: { title: 'Queue', view: queue },
|
||||||
|
oncall: { title: 'On-call', view: oncall },
|
||||||
|
alerts: { title: 'Alerts', view: alerts },
|
||||||
|
more: { title: 'Account', view: account },
|
||||||
|
};
|
||||||
|
|
||||||
|
function parseRoute(pathname) {
|
||||||
|
const m = pathname.match(/^\/incidents\/(\d+)\/?$/);
|
||||||
|
if (m) return { section: 'queue', incident: Number(m[1]) };
|
||||||
|
const name = pathname.replace(/^\/|\/$/g, '');
|
||||||
|
if (name === 'oncall' || name === 'alerts' || name === 'more') return { section: name };
|
||||||
|
return { section: 'queue', incident: null };
|
||||||
|
}
|
||||||
|
|
||||||
|
let route = parseRoute(location.pathname);
|
||||||
|
// How many in-app navigations deep we are, so Back can use the browser's
|
||||||
|
// history when there is somewhere to go back to, and the queue otherwise.
|
||||||
|
let depth = 0;
|
||||||
|
let listScroll = 0;
|
||||||
|
|
||||||
|
export function navigate(path, { replace = false } = {}) {
|
||||||
|
if (path === location.pathname + location.search) return;
|
||||||
|
if (replace) {
|
||||||
|
history.replaceState({ depth }, '', path);
|
||||||
|
} else {
|
||||||
|
depth += 1;
|
||||||
|
history.pushState({ depth }, '', path);
|
||||||
|
}
|
||||||
|
render();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function back() {
|
||||||
|
if (depth > 0) history.back();
|
||||||
|
else navigate('/', { replace: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
window.addEventListener('popstate', (e) => {
|
||||||
|
depth = (e.state && e.state.depth) || 0;
|
||||||
|
render();
|
||||||
|
});
|
||||||
|
|
||||||
|
function render() {
|
||||||
|
const prev = route;
|
||||||
|
route = parseRoute(location.pathname);
|
||||||
|
const app = $('app');
|
||||||
|
|
||||||
|
for (const [name, s] of Object.entries(SECTIONS)) {
|
||||||
|
const el = $(`view-${name}`);
|
||||||
|
el.hidden = name !== route.section;
|
||||||
|
if (name === route.section) $('topbar-title').textContent = s.title;
|
||||||
|
}
|
||||||
|
for (const link of document.querySelectorAll('.nav-link')) {
|
||||||
|
if (link.dataset.section === route.section) link.setAttribute('aria-current', 'page');
|
||||||
|
else link.removeAttribute('aria-current');
|
||||||
|
}
|
||||||
|
|
||||||
|
const detailOpen = route.section === 'queue' && route.incident != null;
|
||||||
|
const wasOpen = prev.section === 'queue' && prev.incident != null;
|
||||||
|
if (detailOpen && !wasOpen) listScroll = window.scrollY;
|
||||||
|
app.classList.toggle('detail-open', detailOpen);
|
||||||
|
$('view-queue').classList.toggle('has-detail', detailOpen);
|
||||||
|
|
||||||
|
if (route.section === 'queue') {
|
||||||
|
queue.show(route.incident);
|
||||||
|
incident.show(route.incident);
|
||||||
|
} else {
|
||||||
|
incident.show(null);
|
||||||
|
SECTIONS[route.section].view.show();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (detailOpen && !wasOpen) window.scrollTo(0, 0);
|
||||||
|
else if (!detailOpen && wasOpen) requestAnimationFrame(() => window.scrollTo(0, listScroll));
|
||||||
|
else if (prev.section !== route.section) window.scrollTo(0, 0);
|
||||||
|
|
||||||
|
updateTitle();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- refresh + badges ----------
|
||||||
|
|
||||||
|
async function refresh() {
|
||||||
|
state.open = await api.incidents({ sort: 'severity' });
|
||||||
|
updateBadges();
|
||||||
|
const jobs = [];
|
||||||
|
if (route.section === 'queue') {
|
||||||
|
jobs.push(queue.refresh());
|
||||||
|
if (route.incident != null) jobs.push(incident.refresh());
|
||||||
|
} else {
|
||||||
|
const v = SECTIONS[route.section].view;
|
||||||
|
if (v.refresh) jobs.push(v.refresh());
|
||||||
|
}
|
||||||
|
await Promise.allSettled(jobs);
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateBadges() {
|
||||||
|
const open = state.open.length;
|
||||||
|
const triggered = state.open.filter((i) => i.status === 'triggered').length;
|
||||||
|
|
||||||
|
const pill = $('open-pill');
|
||||||
|
pill.hidden = false;
|
||||||
|
pill.textContent = open ? `${open} open` : 'All clear';
|
||||||
|
pill.classList.toggle('has-triggered', triggered > 0);
|
||||||
|
pill.classList.toggle('all-acked', open > 0 && triggered === 0);
|
||||||
|
|
||||||
|
const badge = document.querySelector('[data-badge]');
|
||||||
|
badge.hidden = triggered === 0;
|
||||||
|
badge.textContent = String(triggered);
|
||||||
|
updateTitle();
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateTitle() {
|
||||||
|
const triggered = state.open.filter((i) => i.status === 'triggered').length;
|
||||||
|
const section = SECTIONS[route.section].title;
|
||||||
|
const base = route.section === 'queue' && route.incident == null ? 'terdut' : `${section} · terdut`;
|
||||||
|
document.title = triggered ? `(${triggered}) ${base}` : base;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- session ----------
|
||||||
|
|
||||||
|
async function boot() {
|
||||||
|
ui.initSheet();
|
||||||
|
api.setUnauthorizedHandler(showLogin);
|
||||||
|
document.addEventListener('click', interceptLinks);
|
||||||
|
document.addEventListener('keydown', onKey);
|
||||||
|
$('login-form').addEventListener('submit', onLogin);
|
||||||
|
|
||||||
|
try {
|
||||||
|
state.me = await api.me();
|
||||||
|
showApp();
|
||||||
|
} catch (err) {
|
||||||
|
if (err.status === 401) showLogin();
|
||||||
|
else showBootError(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function showBootError(err) {
|
||||||
|
ui.clear($('boot'), ui.emptyState('Cannot load terdut', err.message));
|
||||||
|
$('boot').append(ui.h('button', { class: 'btn', onclick: () => location.reload(), text: 'Retry' }));
|
||||||
|
}
|
||||||
|
|
||||||
|
function showLogin() {
|
||||||
|
poll.stop();
|
||||||
|
ui.closeSheet(null);
|
||||||
|
reset();
|
||||||
|
$('boot').hidden = true;
|
||||||
|
$('app').hidden = true;
|
||||||
|
$('login').hidden = false;
|
||||||
|
const form = $('login-form');
|
||||||
|
form.querySelector('.form-error').hidden = true;
|
||||||
|
form.password.value = '';
|
||||||
|
(form.username.value ? form.password : form.username).focus();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function onLogin(e) {
|
||||||
|
e.preventDefault();
|
||||||
|
const form = e.currentTarget;
|
||||||
|
const err = form.querySelector('.form-error');
|
||||||
|
const btn = form.querySelector('button[type=submit]');
|
||||||
|
err.hidden = true;
|
||||||
|
btn.disabled = true;
|
||||||
|
try {
|
||||||
|
state.me = await api.login(form.username.value.trim(), form.password.value);
|
||||||
|
form.password.value = '';
|
||||||
|
showApp();
|
||||||
|
} catch (ex) {
|
||||||
|
err.textContent = ex.message;
|
||||||
|
err.hidden = false;
|
||||||
|
} finally {
|
||||||
|
btn.disabled = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function signOut() {
|
||||||
|
try {
|
||||||
|
await api.logout();
|
||||||
|
} catch {
|
||||||
|
/* the cookie is cleared server-side or already gone */
|
||||||
|
}
|
||||||
|
showLogin();
|
||||||
|
}
|
||||||
|
|
||||||
|
function showApp() {
|
||||||
|
$('boot').hidden = true;
|
||||||
|
$('login').hidden = true;
|
||||||
|
$('app').hidden = false;
|
||||||
|
render();
|
||||||
|
poll.start(refresh);
|
||||||
|
poll.now();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- links + keys ----------
|
||||||
|
|
||||||
|
function interceptLinks(e) {
|
||||||
|
if (e.defaultPrevented || e.button !== 0 || e.metaKey || e.ctrlKey || e.shiftKey || e.altKey) return;
|
||||||
|
const a = e.target.closest('a[href]');
|
||||||
|
if (!a || a.target || a.origin !== location.origin || a.pathname.startsWith('/api/')) return;
|
||||||
|
e.preventDefault();
|
||||||
|
navigate(a.pathname + a.search);
|
||||||
|
}
|
||||||
|
|
||||||
|
function onKey(e) {
|
||||||
|
if (e.metaKey || e.ctrlKey || e.altKey || ui.sheetIsOpen() || $('app').hidden) return;
|
||||||
|
const tag = e.target.tagName;
|
||||||
|
if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT') return;
|
||||||
|
|
||||||
|
if (e.key === 'r') {
|
||||||
|
poll.now();
|
||||||
|
e.preventDefault();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (route.section !== 'queue') return;
|
||||||
|
if (route.incident != null && incident.key(e)) {
|
||||||
|
e.preventDefault();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (queue.key(e)) e.preventDefault();
|
||||||
|
}
|
||||||
|
|
||||||
|
boot();
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
// Formatting of times, durations and labels.
|
||||||
|
|
||||||
|
const MIN = 60 * 1000;
|
||||||
|
const HOUR = 60 * MIN;
|
||||||
|
const DAY = 24 * HOUR;
|
||||||
|
|
||||||
|
// Compact age for list rows: "now", "4m", "3h", "2d".
|
||||||
|
export function age(iso, now = Date.now()) {
|
||||||
|
const ms = Math.max(0, now - Date.parse(iso));
|
||||||
|
if (ms < MIN) return 'now';
|
||||||
|
if (ms < HOUR) return `${Math.floor(ms / MIN)}m`;
|
||||||
|
if (ms < DAY) return `${Math.floor(ms / HOUR)}h`;
|
||||||
|
return `${Math.floor(ms / DAY)}d`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// "4 min ago", "3 h ago", "yesterday"-free: stays unambiguous at 3am.
|
||||||
|
export function ago(iso, now = Date.now()) {
|
||||||
|
const a = age(iso, now);
|
||||||
|
return a === 'now' ? 'just now' : `${a} ago`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Time remaining until iso, e.g. "1h 20m".
|
||||||
|
export function until(iso, now = Date.now()) {
|
||||||
|
return duration(Date.parse(iso) - now);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function duration(ms) {
|
||||||
|
ms = Math.max(0, ms);
|
||||||
|
if (ms < MIN) return '<1m';
|
||||||
|
const d = Math.floor(ms / DAY);
|
||||||
|
const h = Math.floor((ms % DAY) / HOUR);
|
||||||
|
const m = Math.floor((ms % HOUR) / MIN);
|
||||||
|
if (d) return h ? `${d}d ${h}h` : `${d}d`;
|
||||||
|
if (h) return m ? `${h}h ${m}m` : `${h}h`;
|
||||||
|
return `${m}m`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const timeFmt = new Intl.DateTimeFormat(undefined, { hour: '2-digit', minute: '2-digit' });
|
||||||
|
const dayTimeFmt = new Intl.DateTimeFormat(undefined, {
|
||||||
|
weekday: 'short', day: 'numeric', month: 'short', hour: '2-digit', minute: '2-digit',
|
||||||
|
});
|
||||||
|
|
||||||
|
// Local timestamp; the date is dropped when it is today.
|
||||||
|
export function when(iso) {
|
||||||
|
const d = new Date(iso);
|
||||||
|
const today = new Date();
|
||||||
|
if (d.toDateString() === today.toDateString()) return timeFmt.format(d);
|
||||||
|
return dayTimeFmt.format(d);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isFuture(iso) {
|
||||||
|
return iso != null && Date.parse(iso) > Date.now();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Local calendar dates, as the schedule stores them (YYYY-MM-DD).
|
||||||
|
export function isoDate(d) {
|
||||||
|
const y = d.getFullYear();
|
||||||
|
const m = String(d.getMonth() + 1).padStart(2, '0');
|
||||||
|
const day = String(d.getDate()).padStart(2, '0');
|
||||||
|
return `${y}-${m}-${day}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mondayOf(d) {
|
||||||
|
const r = new Date(d.getFullYear(), d.getMonth(), d.getDate());
|
||||||
|
r.setDate(r.getDate() - ((r.getDay() + 6) % 7));
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function addDays(d, n) {
|
||||||
|
const r = new Date(d);
|
||||||
|
r.setDate(r.getDate() + n);
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ISO 8601 week number.
|
||||||
|
export function isoWeek(d) {
|
||||||
|
const t = new Date(Date.UTC(d.getFullYear(), d.getMonth(), d.getDate()));
|
||||||
|
const day = t.getUTCDay() || 7;
|
||||||
|
t.setUTCDate(t.getUTCDate() + 4 - day);
|
||||||
|
const yearStart = new Date(Date.UTC(t.getUTCFullYear(), 0, 1));
|
||||||
|
return Math.ceil(((t - yearStart) / DAY + 1) / 7);
|
||||||
|
}
|
||||||
|
|
||||||
|
export const STATUS_LABEL = {
|
||||||
|
triggered: 'Triggered',
|
||||||
|
acknowledged: 'Acknowledged',
|
||||||
|
resolved: 'Resolved',
|
||||||
|
snoozed: 'Snoozed',
|
||||||
|
firing: 'Firing',
|
||||||
|
};
|
||||||
|
|
||||||
|
export function severityClass(sev) {
|
||||||
|
const s = (sev || '').toLowerCase();
|
||||||
|
if (s === 'critical' || s === 'page' || s === 'error') return 'sev-critical';
|
||||||
|
if (s === 'warning' || s === 'warn') return 'sev-warning';
|
||||||
|
if (s) return 'sev-info';
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
|
||||||
|
// A one-line summary of the group labels, without the one the title already shows.
|
||||||
|
export function labelSummary(labels, skip = 'alertname') {
|
||||||
|
return Object.entries(labels || {})
|
||||||
|
.filter(([k]) => k !== skip)
|
||||||
|
.map(([k, v]) => `${k}=${v}`)
|
||||||
|
.join(' · ');
|
||||||
|
}
|
||||||
|
|
||||||
|
export function initial(name) {
|
||||||
|
return (name || '?').trim().charAt(0) || '?';
|
||||||
|
}
|
||||||
@@ -0,0 +1,447 @@
|
|||||||
|
// Incident detail: facts, member alerts, the timeline with notes, and the
|
||||||
|
// action bar that carries everything a responder does to an incident.
|
||||||
|
|
||||||
|
import * as api from './api.js';
|
||||||
|
import * as poll from './poll.js';
|
||||||
|
import {
|
||||||
|
h, clear, icon, badge, labelChip, openSheet, closeSheet, confirm, toast, spinner, emptyState,
|
||||||
|
} from './ui.js';
|
||||||
|
import {
|
||||||
|
ago, when, until, isFuture, severityClass, STATUS_LABEL,
|
||||||
|
} from './format.js';
|
||||||
|
import { myID, users } from './state.js';
|
||||||
|
import { back } from './app.js';
|
||||||
|
|
||||||
|
const pane = () => document.getElementById('detail');
|
||||||
|
|
||||||
|
let currentID = null;
|
||||||
|
let inc = null;
|
||||||
|
let events = [];
|
||||||
|
let error = null;
|
||||||
|
let busy = false;
|
||||||
|
|
||||||
|
export function show(id) {
|
||||||
|
if (id === currentID) return;
|
||||||
|
currentID = id;
|
||||||
|
inc = null;
|
||||||
|
events = [];
|
||||||
|
error = null;
|
||||||
|
if (id == null) {
|
||||||
|
renderPlaceholder();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
render();
|
||||||
|
refresh();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function refresh() {
|
||||||
|
const id = currentID;
|
||||||
|
if (id == null) return;
|
||||||
|
try {
|
||||||
|
const [i, t] = await Promise.all([api.incident(id), api.timeline(id)]);
|
||||||
|
if (id !== currentID) return;
|
||||||
|
inc = i;
|
||||||
|
events = t;
|
||||||
|
error = null;
|
||||||
|
} catch (err) {
|
||||||
|
if (id !== currentID) return;
|
||||||
|
error = err.status === 404 ? 'This incident does not exist.' : err.message;
|
||||||
|
}
|
||||||
|
render();
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderPlaceholder() {
|
||||||
|
clear(pane(), h('div', { class: 'detail-placeholder' },
|
||||||
|
h('div', {}, icon('flag', 'icon'), h('p', { text: 'Select an incident to see its alerts and timeline.' }))));
|
||||||
|
}
|
||||||
|
|
||||||
|
function render() {
|
||||||
|
const head = h('div', { class: 'detail-head' },
|
||||||
|
h('button', { class: 'btn btn-ghost btn-icon back', type: 'button', 'aria-label': 'Back to queue', onclick: back },
|
||||||
|
icon('back')),
|
||||||
|
h('span', { class: 'crumb', text: currentID != null ? `Incident #${currentID}` : '' }),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!inc) {
|
||||||
|
clear(pane(), h('div', { class: 'detail' }, head,
|
||||||
|
error ? h('div', { class: 'load-error', text: error }) : spinner()));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep the scroll position across the periodic re-render.
|
||||||
|
const scroller = document.querySelector('.pane-detail');
|
||||||
|
const top = scroller ? scroller.scrollTop : 0;
|
||||||
|
|
||||||
|
clear(pane(),
|
||||||
|
h('article', { class: 'detail' },
|
||||||
|
head,
|
||||||
|
error && h('div', { class: 'load-error', text: `Showing older data: ${error}` }),
|
||||||
|
h('h1', { class: 'detail-title', text: inc.title }),
|
||||||
|
h('div', { class: 'detail-badges' }, statusBadges()),
|
||||||
|
facts(),
|
||||||
|
groupLabels(),
|
||||||
|
alertsSection(),
|
||||||
|
timelineSection(),
|
||||||
|
),
|
||||||
|
actionBar(),
|
||||||
|
);
|
||||||
|
if (scroller) scroller.scrollTop = top;
|
||||||
|
}
|
||||||
|
|
||||||
|
function statusBadges() {
|
||||||
|
const out = [];
|
||||||
|
if (inc.severity) out.push(badge(inc.severity, `plain ${severityClass(inc.severity)}`));
|
||||||
|
out.push(badge(STATUS_LABEL[inc.status] || inc.status, `st-${inc.status}`));
|
||||||
|
if (inc.status !== 'resolved' && isFuture(inc.snoozed_until)) {
|
||||||
|
out.push(badge(`Snoozed · ${until(inc.snoozed_until)} left`, 'st-snoozed'));
|
||||||
|
}
|
||||||
|
if (inc.archived_at) out.push(badge('Archived', 'plain'));
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function who(id, name) {
|
||||||
|
if (id != null && id === myID()) return 'you';
|
||||||
|
return name || 'someone';
|
||||||
|
}
|
||||||
|
|
||||||
|
function facts() {
|
||||||
|
const rows = [];
|
||||||
|
const add = (k, ...v) => rows.push(h('dt', { text: k }), h('dd', {}, ...v));
|
||||||
|
add('Triggered', when(inc.triggered_at), h('span', { class: 'sub', text: ` · ${ago(inc.triggered_at)}` }));
|
||||||
|
if (inc.acknowledged_at) {
|
||||||
|
add('Acknowledged', `${who(inc.acknowledged_by_id, inc.acknowledged_by)} · ${when(inc.acknowledged_at)}`);
|
||||||
|
}
|
||||||
|
add('Assigned', inc.assigned_to_id != null ? who(inc.assigned_to_id, inc.assigned_to) : 'Unassigned');
|
||||||
|
if (inc.status !== 'resolved' && isFuture(inc.snoozed_until)) {
|
||||||
|
add('Snoozed until', when(inc.snoozed_until));
|
||||||
|
}
|
||||||
|
if (inc.resolved_at) {
|
||||||
|
const how = inc.resolution_source === 'manual' ? 'by hand' : 'alerts stopped firing';
|
||||||
|
add('Resolved', when(inc.resolved_at), h('span', { class: 'sub', text: ` · ${how}` }));
|
||||||
|
}
|
||||||
|
if (inc.archived_at) add('Archived', when(inc.archived_at));
|
||||||
|
return h('div', { class: 'card' }, h('dl', { class: 'facts' }, rows));
|
||||||
|
}
|
||||||
|
|
||||||
|
function groupLabels() {
|
||||||
|
const entries = Object.entries(inc.group_labels || {});
|
||||||
|
if (!entries.length) return null;
|
||||||
|
return h('section', { class: 'section' },
|
||||||
|
h('h2', { class: 'section-title', text: 'Grouped by' }),
|
||||||
|
h('div', { class: 'labels-wrap' }, entries.map(([k, v]) => labelChip(k, v))),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function alertsSection() {
|
||||||
|
const list = inc.alerts || [];
|
||||||
|
const firing = list.filter((a) => a.status === 'firing').length;
|
||||||
|
return h('section', { class: 'section' },
|
||||||
|
h('h2', { class: 'section-title' },
|
||||||
|
h('span', { text: `Alerts (${list.length})` }),
|
||||||
|
firing ? h('span', { text: `${firing} firing` }) : null),
|
||||||
|
list.length
|
||||||
|
? h('div', { class: 'card' }, list.map(alertItem))
|
||||||
|
: h('div', { class: 'card card-pad', text: 'No alerts attached.' }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function alertItem(a) {
|
||||||
|
const summary = (a.annotations && (a.annotations.summary || a.annotations.description)) || '';
|
||||||
|
const labels = Object.entries(a.labels || {});
|
||||||
|
return h('div', { class: 'alert-item' },
|
||||||
|
h('div', { class: 'alert-item-head' },
|
||||||
|
h('span', { class: 'alert-item-name', text: a.name }),
|
||||||
|
badge(a.status === 'firing' ? 'Firing' : 'Resolved', `st-${a.status}`)),
|
||||||
|
summary && h('div', { class: 'alert-item-summary', text: summary }),
|
||||||
|
h('div', { class: 'alert-item-foot' },
|
||||||
|
h('span', { text: `Started ${ago(a.starts_at)}` }),
|
||||||
|
h('span', { text: `Last seen ${ago(a.received_at)}` }),
|
||||||
|
a.generator_url && h('a', { href: a.generator_url, target: '_blank', rel: 'noopener noreferrer' }, 'Source ↗'),
|
||||||
|
),
|
||||||
|
labels.length > 0 && h('details', {},
|
||||||
|
h('summary', { text: `${labels.length} labels` }),
|
||||||
|
h('div', { class: 'labels-wrap' }, labels.map(([k, v]) => labelChip(k, v)))),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- timeline ----------
|
||||||
|
|
||||||
|
function eventText(ev) {
|
||||||
|
const person = ev.user_id != null ? who(ev.user_id, ev.username) : null;
|
||||||
|
const strong = (t) => h('span', { class: 'who', text: t || 'someone' });
|
||||||
|
const alertName = () => {
|
||||||
|
const a = (inc.alerts || []).find((x) => x.id === ev.alert_id);
|
||||||
|
return a ? a.name : 'an alert';
|
||||||
|
};
|
||||||
|
switch (ev.type) {
|
||||||
|
case 'triggered': return ['Incident triggered'];
|
||||||
|
case 'alert_added': return [`Alert added: ${alertName()}`];
|
||||||
|
case 'alert_resolved': return [`Alert resolved: ${alertName()}`];
|
||||||
|
case 'acknowledged': return [strong(person), ' acknowledged'];
|
||||||
|
case 'unacknowledged': return [strong(person), ' cleared the acknowledgement'];
|
||||||
|
case 'assigned': return ['Assigned to ', strong(person)];
|
||||||
|
case 'snoozed': return [strong(person), ` snoozed until ${ev.detail ? when(ev.detail) : '…'}`];
|
||||||
|
case 'unsnoozed': return [strong(person), ' ended the snooze'];
|
||||||
|
case 'resolved': return person ? [strong(person), ' resolved the incident'] : ['Resolved: every alert stopped firing'];
|
||||||
|
case 'note': return [strong(person), ' added a note'];
|
||||||
|
case 'notified': {
|
||||||
|
const to = person ? strong(person) : 'the fallback topic';
|
||||||
|
if (ev.detail === 'reminder') return ['Reminder sent to ', to];
|
||||||
|
if (ev.detail === 'resolved') return ['Resolution sent to ', to];
|
||||||
|
return ['Paged ', to];
|
||||||
|
}
|
||||||
|
case 'notify_failed': return ['Notification failed', ev.detail ? `: ${ev.detail}` : ''];
|
||||||
|
case 'deadman_silent': return ['Heartbeat went silent', ev.detail ? ` (${ev.detail})` : ''];
|
||||||
|
default: return [ev.type, ev.detail ? `: ${ev.detail}` : ''];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function timelineSection() {
|
||||||
|
const sorted = [...events].sort((a, b) => Date.parse(a.created_at) - Date.parse(b.created_at) || a.id - b.id);
|
||||||
|
return h('section', { class: 'section' },
|
||||||
|
h('h2', { class: 'section-title' },
|
||||||
|
h('span', { text: 'Timeline' }),
|
||||||
|
h('button', { class: 'btn btn-ghost btn-sm', type: 'button', onclick: addNote },
|
||||||
|
icon('note'), 'Add note')),
|
||||||
|
h('div', { class: 'card' },
|
||||||
|
sorted.length
|
||||||
|
? h('ol', { class: 'timeline' }, sorted.map(timelineItem))
|
||||||
|
: emptyState('No events yet', '')),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function timelineItem(ev) {
|
||||||
|
const mine = ev.type === 'note' && ev.user_id === myID();
|
||||||
|
return h('li', { class: `tl-item tl-${ev.type}` },
|
||||||
|
h('span', { class: 'tl-dot' }),
|
||||||
|
h('div', { class: 'tl-body' },
|
||||||
|
h('div', { class: 'tl-text' }, eventText(ev)),
|
||||||
|
h('div', { class: 'tl-time', title: ev.created_at, text: `${when(ev.created_at)} · ${ago(ev.created_at)}` }),
|
||||||
|
ev.type === 'note' && h('div', { class: 'note', text: ev.detail || '' }),
|
||||||
|
mine && h('div', { class: 'note-actions' },
|
||||||
|
h('button', { class: 'btn btn-ghost btn-sm', type: 'button', onclick: () => deleteNote(ev) }, icon('trash'), 'Delete')),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- actions ----------
|
||||||
|
|
||||||
|
const isOpen = () => inc.status !== 'resolved';
|
||||||
|
const isSnoozed = () => isOpen() && isFuture(inc.snoozed_until);
|
||||||
|
|
||||||
|
function actionBar() {
|
||||||
|
let primary;
|
||||||
|
let secondary;
|
||||||
|
if (inc.status === 'triggered') {
|
||||||
|
primary = h('button', { class: 'btn btn-primary', type: 'button', onclick: acknowledge }, icon('check'), 'Acknowledge');
|
||||||
|
} else if (inc.status === 'acknowledged') {
|
||||||
|
primary = h('button', { class: 'btn btn-primary', type: 'button', onclick: resolve }, icon('checkCircle'), 'Resolve');
|
||||||
|
} else {
|
||||||
|
primary = inc.archived_at
|
||||||
|
? h('button', { class: 'btn btn-primary', type: 'button', onclick: unarchive }, icon('undo'), 'Unarchive')
|
||||||
|
: h('button', { class: 'btn btn-primary', type: 'button', onclick: archive }, icon('archive'), 'Archive');
|
||||||
|
}
|
||||||
|
if (isOpen()) {
|
||||||
|
secondary = isSnoozed()
|
||||||
|
? h('button', { class: 'btn', type: 'button', onclick: unsnooze }, icon('bell'), 'Unsnooze')
|
||||||
|
: h('button', { class: 'btn', type: 'button', onclick: snooze }, icon('clock'), 'Snooze');
|
||||||
|
} else {
|
||||||
|
secondary = h('button', { class: 'btn', type: 'button', onclick: addNote }, icon('note'), 'Note');
|
||||||
|
}
|
||||||
|
const more = h('button', { class: 'btn btn-icon', type: 'button', 'aria-label': 'More actions', onclick: moreMenu }, icon('more'));
|
||||||
|
const bar = h('div', { class: 'actionbar' }, primary, secondary, more);
|
||||||
|
if (busy) for (const b of bar.querySelectorAll('button')) b.disabled = true;
|
||||||
|
return bar;
|
||||||
|
}
|
||||||
|
|
||||||
|
// run performs one action, then reloads the incident and the queue.
|
||||||
|
async function run(fn, done) {
|
||||||
|
if (busy) return;
|
||||||
|
busy = true;
|
||||||
|
render();
|
||||||
|
try {
|
||||||
|
await fn();
|
||||||
|
if (done) toast(done);
|
||||||
|
} catch (err) {
|
||||||
|
toast(err.message, 'error');
|
||||||
|
} finally {
|
||||||
|
busy = false;
|
||||||
|
await refresh();
|
||||||
|
poll.now();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function acknowledge() {
|
||||||
|
const id = inc.id;
|
||||||
|
return run(() => api.acknowledge(id), 'Acknowledged');
|
||||||
|
}
|
||||||
|
|
||||||
|
function unacknowledge() {
|
||||||
|
const id = inc.id;
|
||||||
|
return run(() => api.unacknowledge(id), 'Acknowledgement cleared');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolve() {
|
||||||
|
const id = inc.id;
|
||||||
|
const ok = await confirm({
|
||||||
|
title: 'Resolve this incident?',
|
||||||
|
text: 'Resolving is final. If these alerts fire again they open a new incident, '
|
||||||
|
+ 'and if any are still firing this one stays closed regardless. '
|
||||||
|
+ 'Use snooze if you only need it out of the way.',
|
||||||
|
confirmLabel: 'Resolve',
|
||||||
|
danger: true,
|
||||||
|
});
|
||||||
|
if (ok) await run(() => api.resolve(id), 'Resolved');
|
||||||
|
}
|
||||||
|
|
||||||
|
function archive() {
|
||||||
|
const id = inc.id;
|
||||||
|
return run(() => api.archive(id), 'Archived');
|
||||||
|
}
|
||||||
|
|
||||||
|
function unarchive() {
|
||||||
|
const id = inc.id;
|
||||||
|
return run(() => api.unarchive(id), 'Unarchived');
|
||||||
|
}
|
||||||
|
|
||||||
|
function unsnooze() {
|
||||||
|
const id = inc.id;
|
||||||
|
return run(() => api.unsnooze(id), 'Snooze ended');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function snooze() {
|
||||||
|
const id = inc.id;
|
||||||
|
const tomorrow9 = new Date();
|
||||||
|
tomorrow9.setDate(tomorrow9.getDate() + 1);
|
||||||
|
tomorrow9.setHours(9, 0, 0, 0);
|
||||||
|
|
||||||
|
const options = [
|
||||||
|
['30 minutes', { duration: '30m' }],
|
||||||
|
['1 hour', { duration: '1h' }],
|
||||||
|
['2 hours', { duration: '2h' }],
|
||||||
|
['4 hours', { duration: '4h' }],
|
||||||
|
['8 hours', { duration: '8h' }],
|
||||||
|
['Until 09:00 tomorrow', { until: tomorrow9.toISOString() }],
|
||||||
|
];
|
||||||
|
const spec = await openSheet(() => [
|
||||||
|
h('h2', { class: 'sheet-title', text: 'Snooze' }),
|
||||||
|
h('p', { class: 'sheet-text', text: 'Hide it from the queue for a while. It comes back on its own.' }),
|
||||||
|
h('ul', { class: 'menu' }, options.map(([label, value]) =>
|
||||||
|
h('li', {}, h('button', { class: 'menu-item', type: 'button', onclick: () => closeSheet(value) },
|
||||||
|
icon('clock'), label)))),
|
||||||
|
]);
|
||||||
|
if (spec) await run(() => api.snooze(id, spec), 'Snoozed');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function assign() {
|
||||||
|
const id = inc.id;
|
||||||
|
let list;
|
||||||
|
let onCall;
|
||||||
|
try {
|
||||||
|
[list, onCall] = await Promise.all([users(), api.onCallNow()]);
|
||||||
|
} catch (err) {
|
||||||
|
toast(err.message, 'error');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const me = myID();
|
||||||
|
const sorted = [...list].sort((a, b) => (b.id === me) - (a.id === me) || a.username.localeCompare(b.username));
|
||||||
|
const userID = await openSheet(() => [
|
||||||
|
h('h2', { class: 'sheet-title', text: 'Assign to' }),
|
||||||
|
h('ul', { class: 'menu', role: 'menu' }, sorted.map((u) =>
|
||||||
|
h('li', {}, h('button', {
|
||||||
|
class: 'menu-item',
|
||||||
|
type: 'button',
|
||||||
|
role: 'menuitemradio',
|
||||||
|
'aria-checked': String(u.id === inc.assigned_to_id),
|
||||||
|
onclick: () => closeSheet(u.id),
|
||||||
|
},
|
||||||
|
icon('user'),
|
||||||
|
u.id === me ? `${u.username} (you)` : u.username,
|
||||||
|
onCall && onCall.user_id === u.id ? h('span', { class: 'menu-sub', text: 'on call' }) : null,
|
||||||
|
)))),
|
||||||
|
]);
|
||||||
|
if (userID != null) await run(() => api.assign(id, userID), 'Assigned');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function addNote() {
|
||||||
|
const id = inc.id;
|
||||||
|
const content = await openSheet(() => {
|
||||||
|
const textarea = h('textarea', {
|
||||||
|
name: 'content', required: true, autofocus: true, placeholder: 'What did you find? What did you do?', maxlength: '10000',
|
||||||
|
});
|
||||||
|
const form = h('form', {
|
||||||
|
class: 'sheet-form',
|
||||||
|
onsubmit: (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
const v = textarea.value.trim();
|
||||||
|
if (v) closeSheet(v);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
h('h2', { class: 'sheet-title', text: 'Add note' }),
|
||||||
|
textarea,
|
||||||
|
h('div', { class: 'sheet-actions' },
|
||||||
|
h('button', { class: 'btn', type: 'button', onclick: () => closeSheet(null), text: 'Cancel' }),
|
||||||
|
h('button', { class: 'btn btn-primary', type: 'submit', text: 'Save note' })),
|
||||||
|
);
|
||||||
|
// Ctrl/Cmd+Enter saves, as in most note fields.
|
||||||
|
textarea.addEventListener('keydown', (e) => {
|
||||||
|
if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) form.requestSubmit();
|
||||||
|
});
|
||||||
|
return form;
|
||||||
|
});
|
||||||
|
if (content) await run(() => api.addNote(id, content), 'Note added');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteNote(ev) {
|
||||||
|
const id = inc.id;
|
||||||
|
const ok = await confirm({ title: 'Delete this note?', text: ev.detail || '', confirmLabel: 'Delete', danger: true });
|
||||||
|
if (ok) await run(() => api.deleteNote(id, ev.id), 'Note deleted');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function moreMenu() {
|
||||||
|
const item = (iconName, label, fn, cls = '') =>
|
||||||
|
h('li', {}, h('button', { class: `menu-item ${cls}`, type: 'button', onclick: () => closeSheet(fn) }, icon(iconName), label));
|
||||||
|
|
||||||
|
const items = [];
|
||||||
|
if (isOpen()) {
|
||||||
|
if (inc.status === 'triggered') items.push(item('check', 'Acknowledge', acknowledge));
|
||||||
|
else items.push(item('undo', 'Clear acknowledgement', unacknowledge));
|
||||||
|
items.push(item('user', 'Assign…', assign));
|
||||||
|
items.push(isSnoozed() ? item('bell', 'End snooze', unsnooze) : item('clock', 'Snooze…', snooze));
|
||||||
|
items.push(item('note', 'Add note…', addNote));
|
||||||
|
items.push(h('li', { class: 'menu-sep', role: 'separator' }));
|
||||||
|
items.push(item('checkCircle', 'Resolve…', resolve, 'danger'));
|
||||||
|
} else {
|
||||||
|
items.push(item('note', 'Add note…', addNote));
|
||||||
|
items.push(inc.archived_at ? item('undo', 'Unarchive', unarchive) : item('archive', 'Archive', archive));
|
||||||
|
}
|
||||||
|
|
||||||
|
const fn = await openSheet(() => [
|
||||||
|
h('h2', { class: 'sheet-title', text: inc.title }),
|
||||||
|
h('ul', { class: 'menu' }, items),
|
||||||
|
]);
|
||||||
|
if (fn) await fn();
|
||||||
|
}
|
||||||
|
|
||||||
|
// key handles the detail's shortcuts. Returns true when it used the key.
|
||||||
|
export function key(e) {
|
||||||
|
if (!inc) {
|
||||||
|
if (e.key === 'Escape') {
|
||||||
|
back();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
switch (e.key) {
|
||||||
|
case 'Escape': back(); return true;
|
||||||
|
case 'a': if (inc.status === 'triggered') acknowledge(); return true;
|
||||||
|
case 'A': if (inc.status === 'acknowledged') unacknowledge(); return true;
|
||||||
|
case 'R': if (isOpen()) resolve(); return true;
|
||||||
|
case 's': if (isOpen()) assign(); return true;
|
||||||
|
case 'z': if (isOpen() && !isSnoozed()) snooze(); return true;
|
||||||
|
case 'Z': if (isSnoozed()) unsnooze(); return true;
|
||||||
|
case 'c': addNote(); return true;
|
||||||
|
case 'x': if (!isOpen()) (inc.archived_at ? unarchive() : archive()); return true;
|
||||||
|
default: return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
// On-call: who is on duty now, the week around it, and your own next shifts.
|
||||||
|
// Read-only for now; the TUI edits the schedule.
|
||||||
|
|
||||||
|
import * as api from './api.js';
|
||||||
|
import { h, clear, icon, spinner } from './ui.js';
|
||||||
|
import { isoDate, mondayOf, addDays, isoWeek, initial } from './format.js';
|
||||||
|
import { myID } from './state.js';
|
||||||
|
|
||||||
|
const view = () => document.getElementById('view-oncall');
|
||||||
|
|
||||||
|
let weekStart = mondayOf(new Date());
|
||||||
|
let data = null;
|
||||||
|
let error = null;
|
||||||
|
|
||||||
|
const dayName = new Intl.DateTimeFormat(undefined, { weekday: 'short' });
|
||||||
|
const dayDate = new Intl.DateTimeFormat(undefined, { day: 'numeric', month: 'short' });
|
||||||
|
|
||||||
|
export function show() {
|
||||||
|
if (!data) clear(view(), spinner());
|
||||||
|
refresh();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function refresh() {
|
||||||
|
const start = weekStart;
|
||||||
|
const today = new Date();
|
||||||
|
try {
|
||||||
|
const [now, week, upcoming] = await Promise.all([
|
||||||
|
api.onCallNow(),
|
||||||
|
api.schedule(isoDate(start), isoDate(addDays(start, 6))),
|
||||||
|
api.schedule(isoDate(today), isoDate(addDays(today, 60))),
|
||||||
|
]);
|
||||||
|
if (start !== weekStart) return;
|
||||||
|
data = { now, week, upcoming };
|
||||||
|
error = null;
|
||||||
|
} catch (err) {
|
||||||
|
error = err.message;
|
||||||
|
}
|
||||||
|
render();
|
||||||
|
}
|
||||||
|
|
||||||
|
function shiftWeek(n) {
|
||||||
|
weekStart = addDays(weekStart, 7 * n);
|
||||||
|
refresh();
|
||||||
|
}
|
||||||
|
|
||||||
|
function render() {
|
||||||
|
if (!data) {
|
||||||
|
clear(view(), error ? h('div', { class: 'load-error', text: error }) : spinner());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
clear(view(),
|
||||||
|
error && h('div', { class: 'load-error', text: `Showing older data: ${error}` }),
|
||||||
|
nowCard(),
|
||||||
|
weekCard(),
|
||||||
|
myShifts(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function you(userID) {
|
||||||
|
return userID === myID() ? h('span', { class: 'you', text: 'you' }) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nowCard() {
|
||||||
|
const n = data.now;
|
||||||
|
return h('div', { class: 'card now-card' },
|
||||||
|
h('div', { class: `avatar ${n ? '' : 'none'}`, text: n ? initial(n.username) : '–' }),
|
||||||
|
h('div', {},
|
||||||
|
h('div', { class: 'now-label', text: 'On call now' }),
|
||||||
|
h('div', { class: 'now-name' }, n ? n.username : 'Nobody', n && you(n.user_id)),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function weekCard() {
|
||||||
|
const byDate = new Map(data.week.map((e) => [e.date, e]));
|
||||||
|
const today = isoDate(new Date());
|
||||||
|
const days = [];
|
||||||
|
for (let i = 0; i < 7; i++) {
|
||||||
|
const d = addDays(weekStart, i);
|
||||||
|
const key = isoDate(d);
|
||||||
|
const e = byDate.get(key);
|
||||||
|
days.push(h('li', { class: `day ${key === today ? 'today' : ''} ${key < today ? 'past' : ''}` },
|
||||||
|
h('span', { class: 'day-name', text: dayName.format(d) }),
|
||||||
|
h('span', { class: 'day-date', text: dayDate.format(d) }),
|
||||||
|
h('span', { class: `day-who ${e ? '' : 'nobody'}` }, e ? e.username : 'nobody', e && you(e.user_id)),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
const thisWeek = isoDate(weekStart) === isoDate(mondayOf(new Date()));
|
||||||
|
return [
|
||||||
|
h('div', { class: 'page-head' },
|
||||||
|
h('h2', { text: thisWeek ? 'This week' : 'Week' }),
|
||||||
|
h('div', { class: 'week-nav' },
|
||||||
|
h('button', { class: 'btn btn-ghost btn-icon', type: 'button', 'aria-label': 'Previous week', onclick: () => shiftWeek(-1) },
|
||||||
|
icon('chevronLeft')),
|
||||||
|
h('button', {
|
||||||
|
class: 'btn btn-ghost label',
|
||||||
|
type: 'button',
|
||||||
|
title: 'Back to this week',
|
||||||
|
onclick: () => { weekStart = mondayOf(new Date()); refresh(); },
|
||||||
|
text: `Week ${isoWeek(weekStart)}`,
|
||||||
|
}),
|
||||||
|
h('button', { class: 'btn btn-ghost btn-icon', type: 'button', 'aria-label': 'Next week', onclick: () => shiftWeek(1) },
|
||||||
|
icon('chevronRight')),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
h('ul', { class: 'card days' }, days),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
// myShifts groups your upcoming dates into runs of consecutive days.
|
||||||
|
function myShifts() {
|
||||||
|
const mine = data.upcoming.filter((e) => e.user_id === myID()).map((e) => e.date).sort();
|
||||||
|
const runs = [];
|
||||||
|
for (const date of mine) {
|
||||||
|
const last = runs[runs.length - 1];
|
||||||
|
if (last && isoDate(addDays(parse(last.to), 1)) === date) last.to = date;
|
||||||
|
else runs.push({ from: date, to: date });
|
||||||
|
}
|
||||||
|
const fmt = (s) => `${dayName.format(parse(s))} ${dayDate.format(parse(s))}`;
|
||||||
|
return [
|
||||||
|
h('div', { class: 'page-head' }, h('h2', { text: 'Your next shifts' })),
|
||||||
|
h('div', { class: 'card' },
|
||||||
|
runs.length
|
||||||
|
? h('ul', { class: 'shift-list' }, runs.slice(0, 8).map((r) =>
|
||||||
|
h('li', {},
|
||||||
|
h('span', { text: r.from === r.to ? fmt(r.from) : `${fmt(r.from)} – ${fmt(r.to)}` }),
|
||||||
|
h('span', { class: 'muted', text: days(r) })),
|
||||||
|
))
|
||||||
|
: h('div', { class: 'empty', text: 'Nothing scheduled in the next 60 days.' })),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
function parse(s) {
|
||||||
|
const [y, m, d] = s.split('-').map(Number);
|
||||||
|
return new Date(y, m - 1, d);
|
||||||
|
}
|
||||||
|
|
||||||
|
function days(r) {
|
||||||
|
const n = Math.round((parse(r.to) - parse(r.from)) / 86400000) + 1;
|
||||||
|
return n === 1 ? '1 day' : `${n} days`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
// Keeps the page fresh the way the TUI does: refresh on an interval, but only
|
||||||
|
// while the page is visible, and immediately when it becomes visible again —
|
||||||
|
// which is the moment a phone is picked up after a page.
|
||||||
|
|
||||||
|
const INTERVAL = 20 * 1000;
|
||||||
|
|
||||||
|
let refreshFn = null;
|
||||||
|
let timer = 0;
|
||||||
|
let running = false;
|
||||||
|
let inFlight = null;
|
||||||
|
|
||||||
|
export function start(fn) {
|
||||||
|
refreshFn = fn;
|
||||||
|
running = true;
|
||||||
|
schedule();
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stop() {
|
||||||
|
running = false;
|
||||||
|
clearInterval(timer);
|
||||||
|
}
|
||||||
|
|
||||||
|
// now refreshes straight away and restarts the interval, after an action.
|
||||||
|
export function now() {
|
||||||
|
if (!running) return Promise.resolve();
|
||||||
|
schedule();
|
||||||
|
return tick();
|
||||||
|
}
|
||||||
|
|
||||||
|
function tick() {
|
||||||
|
if (!refreshFn) return Promise.resolve();
|
||||||
|
// Collapse overlapping refreshes into the one already under way.
|
||||||
|
if (!inFlight) {
|
||||||
|
inFlight = Promise.resolve()
|
||||||
|
.then(refreshFn)
|
||||||
|
.catch(() => {})
|
||||||
|
.finally(() => {
|
||||||
|
inFlight = null;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return inFlight;
|
||||||
|
}
|
||||||
|
|
||||||
|
function schedule() {
|
||||||
|
clearInterval(timer);
|
||||||
|
if (running && !document.hidden) timer = setInterval(tick, INTERVAL);
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('visibilitychange', () => {
|
||||||
|
if (!running) return;
|
||||||
|
if (!document.hidden) tick();
|
||||||
|
schedule();
|
||||||
|
});
|
||||||
|
window.addEventListener('online', () => {
|
||||||
|
if (running) tick();
|
||||||
|
});
|
||||||
@@ -0,0 +1,182 @@
|
|||||||
|
// The incident queue: filter chips and a list of incident rows.
|
||||||
|
|
||||||
|
import * as api from './api.js';
|
||||||
|
import { h, clear, badge, emptyState, spinner } from './ui.js';
|
||||||
|
import { age, until, isFuture, severityClass, labelSummary } from './format.js';
|
||||||
|
import { state, myID } from './state.js';
|
||||||
|
import { navigate } from './app.js';
|
||||||
|
|
||||||
|
// The same filters as the TUI's `f` cycle, plus archived ones to get back to.
|
||||||
|
const FILTERS = [
|
||||||
|
{ id: 'open', label: 'Open', query: { sort: 'severity' } },
|
||||||
|
{ id: 'triggered', label: 'Triggered', query: { status: 'triggered', sort: 'severity' } },
|
||||||
|
{ id: 'acknowledged', label: 'Acked', query: { status: 'acknowledged', sort: 'severity' } },
|
||||||
|
{ id: 'snoozed', label: 'Snoozed', query: { snoozed: 'true' } },
|
||||||
|
{ id: 'resolved', label: 'Resolved', query: { status: 'resolved' } },
|
||||||
|
{ id: 'archived', label: 'Archived', query: { status: 'resolved', archived: 'true' } },
|
||||||
|
];
|
||||||
|
|
||||||
|
const EMPTY = {
|
||||||
|
open: ['All clear', 'Nothing open right now.'],
|
||||||
|
triggered: ['Nothing triggered', 'Every open incident has been acknowledged.'],
|
||||||
|
acknowledged: ['Nothing acknowledged', 'No one is working an incident right now.'],
|
||||||
|
snoozed: ['Nothing snoozed', 'Snoozed incidents show up here until the snooze runs out.'],
|
||||||
|
resolved: ['Nothing resolved', 'Resolved incidents are archived after a while.'],
|
||||||
|
archived: ['Nothing archived', ''],
|
||||||
|
};
|
||||||
|
|
||||||
|
let filter = loadFilter();
|
||||||
|
let items = null; // null while loading
|
||||||
|
let error = null;
|
||||||
|
let selected = null;
|
||||||
|
let cursor = -1; // keyboard position in the list
|
||||||
|
let built = false;
|
||||||
|
|
||||||
|
function loadFilter() {
|
||||||
|
try {
|
||||||
|
const f = sessionStorage.getItem('terdut.queue.filter');
|
||||||
|
if (FILTERS.some((x) => x.id === f)) return f;
|
||||||
|
} catch {
|
||||||
|
/* storage unavailable */
|
||||||
|
}
|
||||||
|
return 'open';
|
||||||
|
}
|
||||||
|
|
||||||
|
function saveFilter() {
|
||||||
|
try {
|
||||||
|
sessionStorage.setItem('terdut.queue.filter', filter);
|
||||||
|
} catch {
|
||||||
|
/* storage unavailable */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function show(incidentID) {
|
||||||
|
selected = incidentID;
|
||||||
|
if (!built) {
|
||||||
|
renderChips();
|
||||||
|
built = true;
|
||||||
|
}
|
||||||
|
renderList();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function refresh({ fresh = false } = {}) {
|
||||||
|
const f = FILTERS.find((x) => x.id === filter);
|
||||||
|
const requested = filter;
|
||||||
|
try {
|
||||||
|
// The open list is already fetched for the badges; no need to ask twice.
|
||||||
|
const result = filter === 'open' && !fresh ? state.open : await api.incidents(f.query);
|
||||||
|
if (requested !== filter) return;
|
||||||
|
items = result;
|
||||||
|
error = null;
|
||||||
|
} catch (err) {
|
||||||
|
if (requested !== filter) return;
|
||||||
|
error = err.message;
|
||||||
|
}
|
||||||
|
renderList();
|
||||||
|
}
|
||||||
|
|
||||||
|
function setFilter(id) {
|
||||||
|
if (id === filter) return;
|
||||||
|
filter = id;
|
||||||
|
saveFilter();
|
||||||
|
items = null;
|
||||||
|
cursor = -1;
|
||||||
|
renderChips();
|
||||||
|
renderList();
|
||||||
|
refresh({ fresh: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderChips() {
|
||||||
|
const el = document.getElementById('queue-filters');
|
||||||
|
clear(el, FILTERS.map((f) =>
|
||||||
|
h('button', {
|
||||||
|
class: 'chip',
|
||||||
|
type: 'button',
|
||||||
|
role: 'tab',
|
||||||
|
'aria-selected': String(f.id === filter),
|
||||||
|
onclick: () => setFilter(f.id),
|
||||||
|
text: f.label,
|
||||||
|
}),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderList() {
|
||||||
|
const el = document.getElementById('queue-list');
|
||||||
|
if (error && !items) {
|
||||||
|
clear(el, h('div', { class: 'load-error', text: error }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!items) {
|
||||||
|
clear(el, spinner());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!items.length) {
|
||||||
|
const [title, text] = EMPTY[filter];
|
||||||
|
clear(el, emptyState(title, text, filter === 'open' ? 'checkCircle' : null));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
clear(el,
|
||||||
|
error && h('div', { class: 'load-error', text: `Showing older data: ${error}` }),
|
||||||
|
items.map((inc, i) => row(inc, i)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function row(inc, index) {
|
||||||
|
const snoozed = isFuture(inc.snoozed_until);
|
||||||
|
const resolved = inc.status === 'resolved';
|
||||||
|
|
||||||
|
let status;
|
||||||
|
if (resolved) status = badge('Resolved', 'st-resolved');
|
||||||
|
else if (snoozed) status = badge(`Snoozed · ${until(inc.snoozed_until)}`, 'st-snoozed');
|
||||||
|
else if (inc.status === 'acknowledged') {
|
||||||
|
const by = inc.acknowledged_by_id === myID() ? 'you' : inc.acknowledged_by;
|
||||||
|
status = badge(`Acked${by ? ' · ' + by : ''}`, 'st-acknowledged');
|
||||||
|
}
|
||||||
|
else status = badge('Triggered', 'st-triggered');
|
||||||
|
|
||||||
|
let assignee = null;
|
||||||
|
if (inc.assigned_to_id != null) {
|
||||||
|
assignee = h('span', { text: inc.assigned_to_id === myID() ? '→ you' : `→ ${inc.assigned_to}` });
|
||||||
|
}
|
||||||
|
|
||||||
|
// The server already puts the group labels in the title; show only the rest.
|
||||||
|
const labels = labelSummary(Object.fromEntries(
|
||||||
|
Object.entries(inc.group_labels || {}).filter(([k, v]) => !inc.title.includes(`${k}=${v}`))));
|
||||||
|
return h('a', {
|
||||||
|
class: `row ${severityClass(inc.severity)} ${resolved ? 'resolved' : ''} ${index === cursor ? 'kbd-focus' : ''}`,
|
||||||
|
href: `/incidents/${inc.id}`,
|
||||||
|
'aria-current': inc.id === selected ? 'true' : null,
|
||||||
|
dataset: { index: String(index) },
|
||||||
|
},
|
||||||
|
h('div', { class: 'row-title', text: inc.title }),
|
||||||
|
h('div', { class: 'row-age', title: inc.triggered_at, text: age(inc.triggered_at) }),
|
||||||
|
h('div', { class: 'row-meta' },
|
||||||
|
status,
|
||||||
|
assignee,
|
||||||
|
labels && h('span', { class: 'labels', text: labels }),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// key handles j/k/enter on the list. Returns true when it used the key.
|
||||||
|
export function key(e) {
|
||||||
|
if (!items || !items.length) return false;
|
||||||
|
if (e.key === 'j' || e.key === 'ArrowDown') {
|
||||||
|
cursor = Math.min(items.length - 1, cursor + 1);
|
||||||
|
} else if (e.key === 'k' || e.key === 'ArrowUp') {
|
||||||
|
cursor = Math.max(0, cursor - 1);
|
||||||
|
} else if (e.key === 'Enter' && cursor >= 0) {
|
||||||
|
navigate(`/incidents/${items[cursor].id}`);
|
||||||
|
return true;
|
||||||
|
} else if (e.key === 'f') {
|
||||||
|
const i = FILTERS.findIndex((x) => x.id === filter);
|
||||||
|
setFilter(FILTERS[(i + 1) % FILTERS.length].id);
|
||||||
|
return true;
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
renderList();
|
||||||
|
const el = document.querySelector(`#queue-list [data-index="${cursor}"]`);
|
||||||
|
if (el) el.scrollIntoView({ block: 'nearest' });
|
||||||
|
return true;
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
// State shared between views: who is signed in, the user list, and the open
|
||||||
|
// queue that drives the badges.
|
||||||
|
|
||||||
|
import * as api from './api.js';
|
||||||
|
|
||||||
|
export const state = {
|
||||||
|
me: null, // { user, has_password }
|
||||||
|
open: [], // the default queue: open, not snoozed
|
||||||
|
};
|
||||||
|
|
||||||
|
export function myID() {
|
||||||
|
return state.me ? state.me.user.id : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The user list changes rarely; it is fetched once and then at most every
|
||||||
|
// five minutes, for the assign sheet and for display names.
|
||||||
|
let usersCache = null;
|
||||||
|
let usersAt = 0;
|
||||||
|
export async function users() {
|
||||||
|
if (!usersCache || Date.now() - usersAt > 5 * 60 * 1000) {
|
||||||
|
usersCache = await api.users();
|
||||||
|
usersAt = Date.now();
|
||||||
|
}
|
||||||
|
return usersCache;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function reset() {
|
||||||
|
state.me = null;
|
||||||
|
state.open = [];
|
||||||
|
usersCache = null;
|
||||||
|
}
|
||||||
@@ -0,0 +1,173 @@
|
|||||||
|
// DOM helpers, the bottom sheet, confirmation and toasts.
|
||||||
|
|
||||||
|
// h builds an element. attrs: class, text, on<event>, dataset, aria/other
|
||||||
|
// attributes; boolean true sets an empty attribute, false/null skips it.
|
||||||
|
export function h(tag, attrs = {}, ...children) {
|
||||||
|
const el = document.createElement(tag);
|
||||||
|
for (const [k, v] of Object.entries(attrs || {})) {
|
||||||
|
if (v == null || v === false) continue;
|
||||||
|
if (k === 'class') el.className = v;
|
||||||
|
else if (k === 'text') el.textContent = v;
|
||||||
|
else if (k === 'dataset') Object.assign(el.dataset, v);
|
||||||
|
else if (k.startsWith('on') && typeof v === 'function') el.addEventListener(k.slice(2), v);
|
||||||
|
else if (k in el && typeof v !== 'string') el[k] = v;
|
||||||
|
else el.setAttribute(k, v === true ? '' : v);
|
||||||
|
}
|
||||||
|
append(el, children);
|
||||||
|
return el;
|
||||||
|
}
|
||||||
|
|
||||||
|
function append(el, children) {
|
||||||
|
for (const c of children.flat(Infinity)) {
|
||||||
|
if (c == null || c === false) continue;
|
||||||
|
el.append(c instanceof Node ? c : document.createTextNode(String(c)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clear(el, ...children) {
|
||||||
|
el.replaceChildren();
|
||||||
|
append(el, children);
|
||||||
|
return el;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stroke icons, 24×24. Built as SVG nodes so the CSP needs no inline anything.
|
||||||
|
const ICONS = {
|
||||||
|
back: ['M15 18l-6-6 6-6'],
|
||||||
|
more: ['M5 12h.01M12 12h.01M19 12h.01'],
|
||||||
|
check: ['M5 12.5l4.5 4.5L19 7'],
|
||||||
|
checkCircle: ['M8 12.5l3 3 5-6', 'circle:12,12,9'],
|
||||||
|
undo: ['M9 14L4 9l5-5', 'M4 9h10a6 6 0 0 1 0 12h-3'],
|
||||||
|
user: ['circle:12,8,3.5', 'M5 20a7 7 0 0 1 14 0'],
|
||||||
|
clock: ['circle:12,12,9', 'M12 7v5l3 2'],
|
||||||
|
bell: ['M6 16V11a6 6 0 0 1 12 0v5l1.5 2h-15z', 'M10 20.5a2 2 0 0 0 4 0'],
|
||||||
|
note: ['M5 4h14v12l-4 4H5z', 'M15 20v-4h4', 'M9 9h6M9 13h4'],
|
||||||
|
archive: ['M3.5 5h17v4h-17z', 'M5 9v10h14V9', 'M10 13h4'],
|
||||||
|
flag: ['M5 21V4', 'M5 4h11l-2 4 2 4H5'],
|
||||||
|
trash: ['M4 7h16', 'M9 7V4h6v3', 'M6 7l1 13h10l1-13'],
|
||||||
|
chevronLeft: ['M15 18l-6-6 6-6'],
|
||||||
|
chevronRight: ['M9 6l6 6-6 6'],
|
||||||
|
external: ['M14 4h6v6', 'M20 4l-9 9', 'M18 14v6H4V6h6'],
|
||||||
|
logout: ['M15 4h4v16h-4', 'M10 17l5-5-5-5', 'M15 12H4'],
|
||||||
|
};
|
||||||
|
|
||||||
|
const SVG = 'http://www.w3.org/2000/svg';
|
||||||
|
export function icon(name, cls = 'icon') {
|
||||||
|
const svg = document.createElementNS(SVG, 'svg');
|
||||||
|
svg.setAttribute('viewBox', '0 0 24 24');
|
||||||
|
svg.setAttribute('aria-hidden', 'true');
|
||||||
|
svg.setAttribute('class', cls);
|
||||||
|
for (const d of ICONS[name] || []) {
|
||||||
|
let node;
|
||||||
|
if (d.startsWith('circle:')) {
|
||||||
|
const [cx, cy, r] = d.slice(7).split(',');
|
||||||
|
node = document.createElementNS(SVG, 'circle');
|
||||||
|
node.setAttribute('cx', cx);
|
||||||
|
node.setAttribute('cy', cy);
|
||||||
|
node.setAttribute('r', r);
|
||||||
|
} else {
|
||||||
|
node = document.createElementNS(SVG, 'path');
|
||||||
|
node.setAttribute('d', d);
|
||||||
|
}
|
||||||
|
svg.append(node);
|
||||||
|
}
|
||||||
|
return svg;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- sheet ----------
|
||||||
|
|
||||||
|
const sheet = () => document.getElementById('sheet');
|
||||||
|
let sheetResolve = null;
|
||||||
|
|
||||||
|
// openSheet shows content in the bottom sheet (a centred dialog on desktop)
|
||||||
|
// and resolves with whatever closeSheet is given, or null when dismissed.
|
||||||
|
export function openSheet(build) {
|
||||||
|
const dlg = sheet();
|
||||||
|
if (dlg.open) closeSheet(null);
|
||||||
|
const inner = h('div', { class: 'sheet-inner' }, h('div', { class: 'sheet-grab' }));
|
||||||
|
append(inner, [build()]);
|
||||||
|
clear(dlg, inner);
|
||||||
|
dlg.showModal();
|
||||||
|
const first = dlg.querySelector('[autofocus]');
|
||||||
|
if (first) first.focus();
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
sheetResolve = resolve;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function closeSheet(value = null) {
|
||||||
|
const dlg = sheet();
|
||||||
|
const resolve = sheetResolve;
|
||||||
|
sheetResolve = null;
|
||||||
|
if (dlg.open) dlg.close();
|
||||||
|
if (resolve) resolve(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function sheetIsOpen() {
|
||||||
|
return sheet().open;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function initSheet() {
|
||||||
|
const dlg = sheet();
|
||||||
|
// A tap on the backdrop lands on the dialog element itself.
|
||||||
|
dlg.addEventListener('click', (e) => {
|
||||||
|
if (e.target === dlg) closeSheet(null);
|
||||||
|
});
|
||||||
|
dlg.addEventListener('cancel', (e) => {
|
||||||
|
e.preventDefault();
|
||||||
|
closeSheet(null);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// confirm asks a yes/no question in the sheet.
|
||||||
|
export function confirm({ title, text, confirmLabel = 'Confirm', danger = false }) {
|
||||||
|
return openSheet(() => [
|
||||||
|
h('h2', { class: 'sheet-title', text: title }),
|
||||||
|
text && h('p', { class: 'sheet-text', text }),
|
||||||
|
h('div', { class: 'sheet-actions' },
|
||||||
|
h('button', { class: 'btn', type: 'button', onclick: () => closeSheet(false), text: 'Cancel' }),
|
||||||
|
h('button', {
|
||||||
|
class: `btn ${danger ? 'btn-danger' : 'btn-primary'}`,
|
||||||
|
type: 'button',
|
||||||
|
autofocus: true,
|
||||||
|
onclick: () => closeSheet(true),
|
||||||
|
text: confirmLabel,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
]).then((v) => v === true);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- toast ----------
|
||||||
|
|
||||||
|
let toastTimer = 0;
|
||||||
|
export function toast(message, kind = '') {
|
||||||
|
const el = document.getElementById('toast');
|
||||||
|
el.textContent = message;
|
||||||
|
el.className = `toast ${kind}`;
|
||||||
|
el.hidden = false;
|
||||||
|
clearTimeout(toastTimer);
|
||||||
|
toastTimer = setTimeout(() => {
|
||||||
|
el.hidden = true;
|
||||||
|
}, kind === 'error' ? 5000 : 2500);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- misc ----------
|
||||||
|
|
||||||
|
export function badge(text, cls = '') {
|
||||||
|
return h('span', { class: `badge ${cls}`, text });
|
||||||
|
}
|
||||||
|
|
||||||
|
export function labelChip(k, v) {
|
||||||
|
return h('span', { class: 'label', title: `${k}=${v}` }, h('span', { text: k }), h('span', { text: v }));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function emptyState(title, text, iconName) {
|
||||||
|
return h('div', { class: 'empty' },
|
||||||
|
iconName && icon(iconName),
|
||||||
|
h('strong', { text: title }),
|
||||||
|
text && h('span', { text }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function spinner() {
|
||||||
|
return h('div', { class: 'empty' }, h('span', { class: 'spinner' }));
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"name": "terdut",
|
||||||
|
"short_name": "terdut",
|
||||||
|
"description": "Incident queue and on-call for terdut-server",
|
||||||
|
"start_url": "/",
|
||||||
|
"scope": "/",
|
||||||
|
"display": "standalone",
|
||||||
|
"background_color": "#0f1115",
|
||||||
|
"theme_color": "#1b1e25",
|
||||||
|
"icons": [
|
||||||
|
{ "src": "/icon.svg", "sizes": "any", "type": "image/svg+xml" },
|
||||||
|
{ "src": "/icon-192.png", "sizes": "192x192", "type": "image/png", "purpose": "any" },
|
||||||
|
{ "src": "/icon-512.png", "sizes": "512x512", "type": "image/png", "purpose": "any" },
|
||||||
|
{ "src": "/icon-512.png", "sizes": "512x512", "type": "image/png", "purpose": "maskable" }
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
// Package web serves the web UI, compiled into the binary.
|
||||||
|
//
|
||||||
|
// There is no build step: the files under static/ are what the browser gets.
|
||||||
|
// The page talks to the server's own /api over the same origin, signed in with
|
||||||
|
// the session cookie from POST /api/login.
|
||||||
|
package web
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/sha256"
|
||||||
|
"embed"
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"net/http"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed static
|
||||||
|
var files embed.FS
|
||||||
|
|
||||||
|
// asset is one embedded file, with its validator computed once at startup.
|
||||||
|
type asset struct {
|
||||||
|
body []byte
|
||||||
|
etag string
|
||||||
|
ctype string
|
||||||
|
cache string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handler serves the embedded site. A path without a file extension that
|
||||||
|
// matches no file gets index.html, so a deep link such as /incidents/42 — the
|
||||||
|
// target of a notification tap — survives a reload; the page reads the path
|
||||||
|
// and renders the right view. A missing file with an extension is a real 404.
|
||||||
|
func Handler() (http.Handler, error) {
|
||||||
|
root, err := fs.Sub(files, "static")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
assets := make(map[string]*asset)
|
||||||
|
err = fs.WalkDir(root, ".", func(p string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil || d.IsDir() {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b, err := fs.ReadFile(root, p)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256(b)
|
||||||
|
assets["/"+p] = &asset{
|
||||||
|
body: b,
|
||||||
|
etag: `"` + base64.RawURLEncoding.EncodeToString(sum[:16]) + `"`,
|
||||||
|
ctype: contentType(p),
|
||||||
|
cache: cacheControl(p),
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
index, ok := assets["/index.html"]
|
||||||
|
if !ok {
|
||||||
|
return nil, errors.New("web: static/index.html is missing")
|
||||||
|
}
|
||||||
|
|
||||||
|
// embed.FS reports a zero ModTime, so http.FileServerFS would emit no
|
||||||
|
// validator and every asset would be refetched in full on every load.
|
||||||
|
// Hence the ETag above and ServeContent below, with a zero time that
|
||||||
|
// suppresses Last-Modified.
|
||||||
|
var noTime time.Time
|
||||||
|
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
||||||
|
w.Header().Set("Allow", "GET, HEAD")
|
||||||
|
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p := path.Clean(r.URL.Path)
|
||||||
|
f, ok := assets[p]
|
||||||
|
if !ok {
|
||||||
|
if path.Ext(p) != "" {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
f = index
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", f.ctype)
|
||||||
|
w.Header().Set("Cache-Control", f.cache)
|
||||||
|
w.Header().Set("ETag", f.etag)
|
||||||
|
// The page loads nothing from anywhere else, so the policy can say so
|
||||||
|
// outright rather than carve out exceptions.
|
||||||
|
w.Header().Set("Content-Security-Policy",
|
||||||
|
"default-src 'none'; script-src 'self'; style-src 'self'; img-src 'self' data:; "+
|
||||||
|
"connect-src 'self'; manifest-src 'self'; form-action 'self'; "+
|
||||||
|
"frame-ancestors 'none'; base-uri 'none'")
|
||||||
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||||
|
w.Header().Set("Referrer-Policy", "same-origin")
|
||||||
|
|
||||||
|
http.ServeContent(w, r, "", noTime, bytes.NewReader(f.body))
|
||||||
|
}), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func contentType(p string) string {
|
||||||
|
switch path.Ext(p) {
|
||||||
|
case ".html":
|
||||||
|
return "text/html; charset=utf-8"
|
||||||
|
case ".css":
|
||||||
|
return "text/css; charset=utf-8"
|
||||||
|
case ".js":
|
||||||
|
return "text/javascript; charset=utf-8"
|
||||||
|
case ".svg":
|
||||||
|
return "image/svg+xml"
|
||||||
|
case ".png":
|
||||||
|
return "image/png"
|
||||||
|
case ".webmanifest":
|
||||||
|
return "application/manifest+json"
|
||||||
|
default:
|
||||||
|
return "application/octet-stream"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cacheControl keeps index.html revalidating on every load, because it names
|
||||||
|
// the current asset paths. Assets carry an ETag, so a five-minute window costs
|
||||||
|
// one conditional request after a deploy rather than a stale page.
|
||||||
|
func cacheControl(p string) string {
|
||||||
|
if strings.HasSuffix(p, ".html") {
|
||||||
|
return "no-cache"
|
||||||
|
}
|
||||||
|
return "public, max-age=300"
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user