//go:build migrate // Command sqlite-to-postgres copies a terdut SQLite database into a freshly // migrated Postgres one. It exists for exactly one upgrade — the one that moved // this server off SQLite — and should be deleted once the installs that need it // have run it. The modernc.org/sqlite dependency goes with it. // // Build-tagged so the dependency stays out of the server binary and out of a // plain `go build ./...`: // // go run -tags migrate ./scripts/sqlite-to-postgres.go \ // -sqlite /path/to/terdut.db \ // -dsn 'postgres://terdut:secret@localhost:5432/terdut?sslmode=disable' // // The Postgres side must already have the schema: start the new server once // against an empty database, let it migrate, stop it, then run this. The copy // refuses to touch a database that already has rows, so a second run cannot // double-insert. // // Ids are preserved, which is what keeps every foreign key — incident_alerts, // incident_events, notifications, the ack tokens — pointing at the same rows it // pointed at before. The identity sequences are moved past the copied ids at the // end, so the first row the server writes afterwards does not collide. package main import ( "database/sql" "flag" "fmt" "log" "strings" _ "github.com/jackc/pgx/v5/stdlib" _ "modernc.org/sqlite" ) // tables are copied parents first: every foreign key points at a table earlier // in this list. var tables = []struct { name string columns []string // jsonb marks columns that were TEXT in SQLite and are jsonb in Postgres, // so the insert can cast them. jsonb []string // sequence is the identity sequence to advance afterwards, empty when the // table has no generated id. sequence string }{ {name: "users", columns: []string{"id", "username", "email", "created_at", "ntfy_topic", "password_hash"}, sequence: "users_id_seq"}, {name: "api_keys", columns: []string{"id", "user_id", "key_hash", "name", "created_at", "last_used_at"}, sequence: "api_keys_id_seq"}, {name: "sessions", columns: []string{"id", "token_hash", "user_id", "created_at", "last_seen_at", "expires_at", "user_agent"}, sequence: "sessions_id_seq"}, {name: "alerts", columns: []string{"id", "fingerprint", "name", "status", "labels", "annotations", "starts_at", "ends_at", "generator_url", "received_at", "archived_at", "resolution_source"}, jsonb: []string{"labels", "annotations"}, sequence: "alerts_id_seq"}, {name: "schedule_entries", columns: []string{"id", "user_id", "date", "created_at"}, sequence: "schedule_entries_id_seq"}, {name: "incidents", columns: []string{"id", "group_key", "title", "group_labels", "status", "severity", "triggered_at", "acknowledged_by", "acknowledged_at", "assigned_to", "snoozed_until", "resolved_at", "resolution_source", "archived_at"}, jsonb: []string{"group_labels"}, sequence: "incidents_id_seq"}, {name: "incident_alerts", columns: []string{"incident_id", "alert_id", "added_at"}}, {name: "incident_events", columns: []string{"id", "incident_id", "type", "user_id", "alert_id", "detail", "created_at"}, sequence: "incident_events_id_seq"}, {name: "notifications", columns: []string{"id", "incident_id", "user_id", "topic", "kind", "created_at", "send_after", "attempts", "sent_at", "last_error"}, sequence: "notifications_id_seq"}, {name: "incident_ack_tokens", columns: []string{"token_hash", "incident_id", "user_id", "created_at", "expires_at"}}, } func main() { sqlitePath := flag.String("sqlite", "", "path to the existing terdut SQLite database") dsn := flag.String("dsn", "", "Postgres DSN of the migrated, empty database") flag.Parse() if *sqlitePath == "" || *dsn == "" { log.Fatal("both -sqlite and -dsn are required") } src, err := sql.Open("sqlite", *sqlitePath) if err != nil { log.Fatalf("open sqlite: %v", err) } defer src.Close() dst, err := sql.Open("pgx", *dsn) if err != nil { log.Fatalf("open postgres: %v", err) } defer dst.Close() if err := dst.Ping(); err != nil { log.Fatalf("ping postgres: %v", err) } if err := assertEmpty(dst); err != nil { log.Fatalf("%v", err) } // One transaction for the whole copy: a run that dies half way leaves the // target as it found it, rather than a partial database someone has to // recognise as partial. tx, err := dst.Begin() if err != nil { log.Fatalf("begin: %v", err) } defer tx.Rollback() //nolint:errcheck for _, t := range tables { n, err := copyTable(src, tx, t.name, t.columns, t.jsonb) if err != nil { log.Fatalf("copy %s: %v", t.name, err) } log.Printf("%-20s %d row(s)", t.name, n) } for _, t := range tables { if t.sequence == "" { continue } if err := advanceSequence(tx, t.sequence, t.name); err != nil { log.Fatalf("advance %s: %v", t.sequence, err) } } if err := tx.Commit(); err != nil { log.Fatalf("commit: %v", err) } log.Print("done") } // assertEmpty refuses a target that already holds data, so running this twice // cannot duplicate anything. func assertEmpty(dst *sql.DB) error { for _, t := range tables { var n int64 if err := dst.QueryRow("SELECT COUNT(*) FROM " + t.name).Scan(&n); err != nil { return fmt.Errorf("count %s (has the new server migrated this database?): %w", t.name, err) } if n > 0 { return fmt.Errorf("%s already has %d row(s): the target must be empty", t.name, n) } } return nil } func copyTable(src *sql.DB, tx *sql.Tx, table string, columns, jsonb []string) (int64, error) { rows, err := src.Query("SELECT " + strings.Join(columns, ", ") + " FROM " + table) if err != nil { return 0, err } defer rows.Close() insert := "INSERT INTO " + table + " (" + strings.Join(columns, ", ") + ") VALUES (" + strings.Join(valuePlaceholders(columns, jsonb), ", ") + ")" stmt, err := tx.Prepare(insert) if err != nil { return 0, err } defer stmt.Close() var copied int64 for rows.Next() { // Scanning into any lets the SQLite driver decide each column's Go type // and hands it straight back to pgx, which is all this needs: the column // types match on both sides, apart from the JSON casts above. values := make([]any, len(columns)) targets := make([]any, len(columns)) for i := range values { targets[i] = &values[i] } if err := rows.Scan(targets...); err != nil { return copied, err } if _, err := stmt.Exec(values...); err != nil { return copied, fmt.Errorf("insert row %d: %w", copied+1, err) } copied++ } return copied, rows.Err() } // valuePlaceholders numbers the placeholders, casting the columns that became // jsonb: pgx sends a Go string as text, and Postgres will not assign text to a // jsonb column without being told. func valuePlaceholders(columns, jsonb []string) []string { isJSON := make(map[string]bool, len(jsonb)) for _, c := range jsonb { isJSON[c] = true } out := make([]string, len(columns)) for i, c := range columns { out[i] = fmt.Sprintf("$%d", i+1) if isJSON[c] { out[i] += "::jsonb" } } return out } // advanceSequence puts an identity sequence past the largest copied id. Without // it the first insert after the migration would reuse id 1. func advanceSequence(tx *sql.Tx, sequence, table string) error { _, err := tx.Exec(fmt.Sprintf( `SELECT setval('%s', COALESCE((SELECT MAX(id) FROM %s), 0) + 1, false)`, sequence, table)) return err }