Retry the first database ping instead of dying on it
Every start crashed once or twice before going healthy: kube-router enforces this namespace's NetworkPolicy per-node, reacting to the new pod's creation event, and the app's first connection attempt can reach Postgres's node before that node's allow-set has been updated with the new pod's IP. The result is "connection refused" -- an active reject, not a timeout, which is how it was told apart from Postgres itself not being ready (it had been up for two days in the run that was diagnosed). That race resolves within several seconds in practice, so Open now retries the ping up to five times, two seconds apart, logging each failure, before giving up with the same wrapped error as before. Nothing else about Open's behaviour changed: a genuinely absent database still fails, just after ~8s instead of immediately. Claude-Session: https://claude.ai/code/session_01RHPj4ggeFdEjKKfm4SHbD7
This commit is contained in:
+24
-2
@@ -5,6 +5,7 @@ import (
|
|||||||
"embed"
|
"embed"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
|
"log"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -15,6 +16,19 @@ import (
|
|||||||
//go:embed migrations
|
//go:embed migrations
|
||||||
var migrationsFS embed.FS
|
var migrationsFS embed.FS
|
||||||
|
|
||||||
|
// pingAttempts and pingRetryDelay bound the retry on the first connection.
|
||||||
|
// This pod's own IP can reach the Postgres pod's node before that node's
|
||||||
|
// NetworkPolicy enforcement (kube-router, reacting to the pod's creation
|
||||||
|
// event) has added it to the allowed-source set, which fails the ping with
|
||||||
|
// "connection refused" rather than a timeout. That race resolves within
|
||||||
|
// several seconds in practice; five attempts two seconds apart give it
|
||||||
|
// comfortable room without turning a genuinely absent database into a long
|
||||||
|
// hang.
|
||||||
|
const (
|
||||||
|
pingAttempts = 5
|
||||||
|
pingRetryDelay = 2 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
// Open connects to Postgres. dsn is a libpq connection string or URL, e.g.
|
// Open connects to Postgres. dsn is a libpq connection string or URL, e.g.
|
||||||
// postgres://terdut:secret@localhost:5432/terdut?sslmode=disable.
|
// postgres://terdut:secret@localhost:5432/terdut?sslmode=disable.
|
||||||
//
|
//
|
||||||
@@ -33,11 +47,19 @@ func Open(dsn string) (*sql.DB, error) {
|
|||||||
db.SetMaxOpenConns(10)
|
db.SetMaxOpenConns(10)
|
||||||
db.SetMaxIdleConns(5)
|
db.SetMaxIdleConns(5)
|
||||||
db.SetConnMaxLifetime(time.Hour)
|
db.SetConnMaxLifetime(time.Hour)
|
||||||
if err := db.Ping(); err != nil {
|
|
||||||
|
for attempt := 1; ; attempt++ {
|
||||||
|
err = db.Ping()
|
||||||
|
if err == nil {
|
||||||
|
return db, nil
|
||||||
|
}
|
||||||
|
if attempt == pingAttempts {
|
||||||
db.Close()
|
db.Close()
|
||||||
return nil, fmt.Errorf("ping: %w", err)
|
return nil, fmt.Errorf("ping: %w", err)
|
||||||
}
|
}
|
||||||
return db, nil
|
log.Printf("open db: ping attempt %d/%d failed, retrying in %s: %v", attempt, pingAttempts, pingRetryDelay, err)
|
||||||
|
time.Sleep(pingRetryDelay)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Migrate applies every embedded migration that has not been applied yet, in
|
// Migrate applies every embedded migration that has not been applied yet, in
|
||||||
|
|||||||
Reference in New Issue
Block a user