feat(broker): migrations-based install, roles, RLS, and job dependency groups
Replace the ad-hoc tables/procedures install layout with versioned, ordered SQL migrations tracked in broker_schema_migrations. Add optional least-privilege role provisioning (--with-roles), multi-tenant row-level security, lease-based job claiming with stale-lease recovery, and job dependencies -- both by job id and by fan-in job group. Add Docker/Compose support for running the broker and its test suite.
This commit is contained in:
@@ -19,6 +19,11 @@ type DBAdapter interface {
|
||||
// Begin starts a new transaction
|
||||
Begin(ctx context.Context) (DBTransaction, error)
|
||||
|
||||
// Conn returns a single physical connection pinned out of the pool, for
|
||||
// session-scoped state (e.g. advisory locks) that must survive across
|
||||
// calls. The caller owns it and must Close() it when done.
|
||||
Conn(ctx context.Context) (*sql.Conn, error)
|
||||
|
||||
// Exec executes a query without returning rows
|
||||
Exec(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
|
||||
|
||||
|
||||
@@ -192,8 +192,11 @@ func (p *PostgresAdapter) Listen(ctx context.Context, channel string, handler No
|
||||
|
||||
p.logger.Info("listening on channel", "channel", channel)
|
||||
|
||||
// Start notification handler in goroutine
|
||||
go func() {
|
||||
// Start notification handler in a supervised goroutine: it must keep
|
||||
// running for the life of the process, so a panic (e.g. from a
|
||||
// misbehaving handler) is logged and the loop restarted rather than
|
||||
// silently dying.
|
||||
SupervisedGo(p.logger, "listener-"+channel, func() {
|
||||
for {
|
||||
select {
|
||||
case n := <-listener.Notify:
|
||||
@@ -208,10 +211,10 @@ func (p *PostgresAdapter) Listen(ctx context.Context, channel string, handler No
|
||||
p.logger.Info("stopping listener", "channel", channel)
|
||||
return
|
||||
case <-time.After(90 * time.Second):
|
||||
go listener.Ping()
|
||||
SafeGo(p.logger, "listener-ping-"+channel, func() { listener.Ping() })
|
||||
}
|
||||
}
|
||||
}()
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -237,7 +240,7 @@ func (p *PostgresAdapter) buildConnectionString() string {
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
|
||||
"host=%s port=%d user=%s password=%s dbname=%s sslmode=%s options='-c search_path=broker,public'",
|
||||
p.config.Host,
|
||||
p.config.Port,
|
||||
p.config.User,
|
||||
@@ -247,6 +250,22 @@ func (p *PostgresAdapter) buildConnectionString() string {
|
||||
)
|
||||
}
|
||||
|
||||
// Conn returns a single physical connection pinned out of the pool, for
|
||||
// session-scoped operations (e.g. pg_try_advisory_lock) that must survive
|
||||
// across calls and must not be silently reaped or handed to another caller
|
||||
// by the pool. The caller owns its lifecycle and must Close() it.
|
||||
func (p *PostgresAdapter) Conn(ctx context.Context) (*sql.Conn, error) {
|
||||
p.mu.RLock()
|
||||
db := p.db
|
||||
p.mu.RUnlock()
|
||||
|
||||
if db == nil {
|
||||
return nil, fmt.Errorf("database connection not established")
|
||||
}
|
||||
|
||||
return db.Conn(ctx)
|
||||
}
|
||||
|
||||
// postgresTransaction implements DBTransaction
|
||||
type postgresTransaction struct {
|
||||
tx *sql.Tx
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package adapter
|
||||
|
||||
import (
|
||||
"runtime/debug"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RecoverAndLog recovers a panic (if any) and logs it with a stack trace.
|
||||
// Call it via `defer adapter.RecoverAndLog(logger, "name")` at the top of
|
||||
// any goroutine body that must never be allowed to crash the process.
|
||||
func RecoverAndLog(logger Logger, name string) {
|
||||
if r := recover(); r != nil {
|
||||
logger.Error("recovered from panic", "component", name, "panic", r, "stack", string(debug.Stack()))
|
||||
}
|
||||
}
|
||||
|
||||
// SafeGo runs fn in a new goroutine, recovering any panic so it can never
|
||||
// crash the process. Use for one-shot/fire-and-forget goroutines; a panic is
|
||||
// logged (with its stack trace) and the goroutine simply ends.
|
||||
func SafeGo(logger Logger, name string, fn func()) {
|
||||
go func() {
|
||||
defer RecoverAndLog(logger, name)
|
||||
fn()
|
||||
}()
|
||||
}
|
||||
|
||||
// SupervisedGo runs fn in a new goroutine. If fn panics, the panic is logged
|
||||
// and fn is restarted (after a short backoff) instead of letting the
|
||||
// goroutine die permanently. Use for long-running loops (ticker routines,
|
||||
// notification listeners, worker loops) that must keep running for the life
|
||||
// of the process. fn must return normally, without panicking, once its own
|
||||
// shutdown/context-done condition is met -- a clean return is not restarted.
|
||||
func SupervisedGo(logger Logger, name string, fn func()) {
|
||||
go func() {
|
||||
for {
|
||||
if runSupervised(logger, name, fn) {
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// runSupervised runs fn once, recovering a panic if it occurs. It returns
|
||||
// true if fn returned normally (no restart needed) and false if it panicked
|
||||
// (caller should restart it).
|
||||
func runSupervised(logger Logger, name string, fn func()) (clean bool) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
logger.Error("recovered from panic, restarting", "component", name, "panic", r, "stack", string(debug.Stack()))
|
||||
clean = false
|
||||
}
|
||||
}()
|
||||
fn()
|
||||
return true
|
||||
}
|
||||
Reference in New Issue
Block a user