terdut-server/internal/api/metrics_test.go

package api_test

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"sync/atomic"
	"testing"

	"git.ryuvia.com/niklas/terdut-server/internal/api"
)

// fakePrometheus answers the two query endpoints and counts the requests.
func fakePrometheus(t *testing.T) (*httptest.Server, *atomic.Int32) {
	t.Helper()
	var hits atomic.Int32
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		hits.Add(1)
		w.Header().Set("Content-Type", "application/json")
		switch r.URL.Path {
		case "/api/v1/query_range":
			w.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[
				{"metric":{"instance":"web-1"},"values":[[100,"1"],[160,"2"]]},
				{"metric":{"instance":"web-2"},"values":[[100,"9"]]}]}}`))
		case "/api/v1/query":
			w.Write([]byte(`{"status":"success","data":{"resultType":"vector","result":[
				{"metric":{"instance":"web-1"},"value":[170,"3"]}]}}`))
		default:
			w.WriteHeader(404)
		}
	}))
	t.Cleanup(srv.Close)
	return srv, &hits
}

type metricsBody struct {
	Enabled bool `json:"enabled"`
	Alerts  []struct {
		AlertID   int64  `json:"alert_id"`
		Expr      string `json:"expr"`
		Available bool   `json:"available"`
		Reason    string `json:"reason"`
		Snapshots map[string]struct {
			Series json.RawMessage `json:"series"`
			Error  string          `json:"error"`
		} `json:"snapshots"`
		Live *struct {
			Series  []struct{ Labels map[string]string } `json:"series"`
			Current []struct{ Labels map[string]string } `json:"current"`
		} `json:"live"`
	} `json:"alerts"`
}

func getMetrics(t *testing.T, s *ts, id int) metricsBody {
	t.Helper()
	var out metricsBody
	decode(t, s.req(t, http.MethodGet, "/api/incidents/"+itoa(id)+"/metrics", nil), &out)
	return out
}

func itoa(n int) string {
	b, _ := json.Marshal(n)
	return string(b)
}

func metricsAlert(promURL string) map[string]any {
	a := amAlert("fp-metrics", "HighLatency", "firing", "2026-05-20T10:00:00Z", "0001-01-01T00:00:00Z",
		map[string]string{"instance": "web-1"})
	a["generatorURL"] = promURL + "/graph?g0.expr=latency%3E1&g0.tab=1"
	return a
}

func TestMetrics_SnapshotsOpenedAndClosedAndLive(t *testing.T) {
	prom, _ := fakePrometheus(t)
	conf := testConfig()
	conf.PrometheusAllowedHosts = []string{"127.0.0.0/8"} // httptest listens on loopback
	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)

	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-metrics")
	incs := listIncidents(t, s, "")
	if len(incs) != 1 {
		t.Fatalf("want one incident, got %d", len(incs))
	}
	id := int(incs[0]["id"].(float64))

	api.MetricsSweep(context.Background(), s.db, conf)

	m := getMetrics(t, s, id)
	if !m.Enabled || len(m.Alerts) != 1 {
		t.Fatalf("metrics = %+v", m)
	}
	a := m.Alerts[0]
	if !a.Available || a.Expr != "latency>1" {
		t.Fatalf("alert = %+v", a)
	}
	opened, ok := a.Snapshots["opened"]
	if !ok || opened.Error != "" {
		t.Fatalf("opened snapshot = %+v", a.Snapshots)
	}
	var series []struct{ Labels map[string]string }
	json.Unmarshal(opened.Series, &series)
	if len(series) != 1 || series[0].Labels["instance"] != "web-1" {
		t.Errorf("snapshot should hold only this alert's series (web-1), got %+v", series)
	}
	if _, ok := a.Snapshots["closed"]; ok {
		t.Error("closed snapshot exists while the incident is open")
	}
	if a.Live == nil || len(a.Live.Series) != 1 || len(a.Live.Current) != 1 {
		t.Errorf("an open incident should have live data, got %+v", a.Live)
	}

	// A second pass must not capture again.
	var n int
	s.db.QueryRow("SELECT count(*) FROM incident_metric_snapshots").Scan(&n)
	api.MetricsSweep(context.Background(), s.db, conf)
	var n2 int
	s.db.QueryRow("SELECT count(*) FROM incident_metric_snapshots").Scan(&n2)
	if n != 1 || n2 != 1 {
		t.Errorf("snapshot rows = %d then %d, want 1 and 1", n, n2)
	}

	// Resolving the incident gets the closed snapshot, and no more live data.
	resp := s.req(t, http.MethodPost, "/api/incidents/"+itoa(id)+"/resolve", nil)
	resp.Body.Close()
	api.MetricsSweep(context.Background(), s.db, conf)
	m = getMetrics(t, s, id)
	a = m.Alerts[0]
	if _, ok := a.Snapshots["closed"]; !ok {
		t.Errorf("no closed snapshot after resolve: %+v", a.Snapshots)
	}
	if a.Live != nil {
		t.Error("a resolved incident should not query live")
	}
}

func TestMetrics_DisabledWithoutAllowedHosts(t *testing.T) {
	prom, hits := fakePrometheus(t)
	s := newTS(t)
	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-off")
	id := int(listIncidents(t, s, "")[0]["id"].(float64))

	api.MetricsSweep(context.Background(), s.db, testConfig())
	m := getMetrics(t, s, id)
	if m.Enabled || len(m.Alerts) != 0 {
		t.Errorf("metrics = %+v, want disabled and empty", m)
	}
	if hits.Load() != 0 {
		t.Errorf("Prometheus was contacted %d times", hits.Load())
	}
}

func TestMetrics_UnlistedHostIsRecordedNotQueried(t *testing.T) {
	prom, hits := fakePrometheus(t)
	conf := testConfig()
	conf.PrometheusAllowedHosts = []string{"*.example.com"}
	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)
	postWebhook(t, s, []map[string]any{metricsAlert(prom.URL)}, "g-unlisted")
	id := int(listIncidents(t, s, "")[0]["id"].(float64))

	api.MetricsSweep(context.Background(), s.db, conf)
	api.MetricsSweep(context.Background(), s.db, conf) // must not retry a permanent failure

	m := getMetrics(t, s, id)
	a := m.Alerts[0]
	if a.Available || a.Reason == "" {
		t.Errorf("alert should be unavailable with a reason: %+v", a)
	}
	if hits.Load() != 0 {
		t.Errorf("an unlisted host was contacted %d times", hits.Load())
	}
	var attempts int
	var errText *string
	s.db.QueryRow("SELECT attempts, error FROM incident_metric_snapshots").Scan(&attempts, &errText)
	if errText == nil || attempts < 4 {
		t.Errorf("failure should be recorded as final: attempts=%d error=%v", attempts, errText)
	}
}

func TestMetrics_IncidentScoping(t *testing.T) {
	conf := testConfig()
	conf.PrometheusAllowedHosts = []string{"127.0.0.0/8"}
	s := newTSWith(t, api.DeadmanConfig{}, api.NotifyConfig{}, conf)
	resp := s.req(t, http.MethodGet, "/api/incidents/9999/metrics", nil)
	resp.Body.Close()
	if resp.StatusCode != http.StatusNotFound {
		t.Errorf("unknown incident: %d, want 404", resp.StatusCode)
	}
}