feat(broker): migrations-based install, roles, RLS, and job dependency groups
Replace the ad-hoc tables/procedures install layout with versioned, ordered SQL migrations tracked in broker_schema_migrations. Add optional least-privilege role provisioning (--with-roles), multi-tenant row-level security, lease-based job claiming with stale-lease recovery, and job dependencies -- both by job id and by fan-in job group. Add Docker/Compose support for running the broker and its test suite.
This commit is contained in:
+113
-50
@@ -2,13 +2,13 @@ package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql" // Import sql package
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
|
||||
"git.warky.dev/wdevs/pgsql-broker/pkg/broker/models"
|
||||
)
|
||||
|
||||
// Worker represents a single job processing worker
|
||||
@@ -18,7 +18,7 @@ type Worker struct {
|
||||
InstanceID int64
|
||||
db adapter.DBAdapter
|
||||
logger adapter.Logger
|
||||
jobChan chan models.Job
|
||||
wakeChan chan struct{}
|
||||
shutdown chan struct{}
|
||||
wg *sync.WaitGroup
|
||||
running bool
|
||||
@@ -27,6 +27,8 @@ type Worker struct {
|
||||
jobsHandled int64
|
||||
timerSeconds int
|
||||
fetchSize int
|
||||
tenantID string
|
||||
leaseSeconds int
|
||||
}
|
||||
|
||||
// Stats holds worker statistics
|
||||
@@ -46,21 +48,30 @@ type Config struct {
|
||||
BufferSize int
|
||||
TimerSeconds int
|
||||
FetchSize int
|
||||
TenantID string
|
||||
LeaseSeconds int
|
||||
}
|
||||
|
||||
// New creates a new worker
|
||||
func New(cfg Config) *Worker {
|
||||
leaseSeconds := cfg.LeaseSeconds
|
||||
if leaseSeconds <= 0 {
|
||||
leaseSeconds = 60
|
||||
}
|
||||
|
||||
return &Worker{
|
||||
ID: cfg.ID,
|
||||
QueueNumber: cfg.QueueNumber,
|
||||
InstanceID: cfg.InstanceID,
|
||||
db: cfg.DBAdapter,
|
||||
logger: cfg.Logger.With("worker_id", cfg.ID).With("queue", cfg.QueueNumber),
|
||||
jobChan: make(chan models.Job, cfg.BufferSize),
|
||||
wakeChan: make(chan struct{}, 1),
|
||||
shutdown: make(chan struct{}),
|
||||
wg: &sync.WaitGroup{},
|
||||
timerSeconds: cfg.TimerSeconds,
|
||||
fetchSize: cfg.FetchSize,
|
||||
tenantID: cfg.TenantID,
|
||||
leaseSeconds: leaseSeconds,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,11 +88,48 @@ func (w *Worker) Start(ctx context.Context) error {
|
||||
w.logger.Info("worker starting")
|
||||
|
||||
w.wg.Add(1)
|
||||
go w.processLoop(ctx)
|
||||
go w.superviseProcessLoop(ctx)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// superviseProcessLoop runs processLoop for the life of the worker,
|
||||
// restarting it (after a short backoff) if it ever panics, so a bug in job
|
||||
// processing can never permanently kill this worker's goroutine.
|
||||
func (w *Worker) superviseProcessLoop(ctx context.Context) {
|
||||
defer w.wg.Done()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-w.shutdown:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
if w.runProcessLoopOnce(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// runProcessLoopOnce runs processLoop, recovering any panic. It returns true
|
||||
// if processLoop returned normally (shutdown/context done, no restart
|
||||
// needed) and false if it panicked (caller should restart it).
|
||||
func (w *Worker) runProcessLoopOnce(ctx context.Context) (clean bool) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
w.logger.Error("worker panic recovered, restarting", "panic", r, "stack", string(debug.Stack()))
|
||||
clean = false
|
||||
}
|
||||
}()
|
||||
w.processLoop(ctx)
|
||||
return true
|
||||
}
|
||||
|
||||
// Stop gracefully stops the worker
|
||||
func (w *Worker) Stop() error {
|
||||
w.mu.Lock()
|
||||
@@ -103,35 +151,33 @@ func (w *Worker) Stop() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddJob adds a job to the worker's queue
|
||||
func (w *Worker) AddJob(job models.Job) error {
|
||||
// Wake signals the worker to check for available jobs immediately, instead
|
||||
// of waiting for the next timer tick. Fetching is always DB-driven (via
|
||||
// broker_get with FOR UPDATE SKIP LOCKED), so a redundant or coalesced wake
|
||||
// is harmless.
|
||||
func (w *Worker) Wake() {
|
||||
select {
|
||||
case w.jobChan <- job:
|
||||
return nil
|
||||
case w.wakeChan <- struct{}{}:
|
||||
default:
|
||||
return fmt.Errorf("worker %d job channel is full", w.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// processLoop is the main worker processing loop
|
||||
func (w *Worker) processLoop(ctx context.Context) {
|
||||
defer w.wg.Done()
|
||||
defer w.recoverPanic()
|
||||
|
||||
timer := time.NewTimer(time.Duration(w.timerSeconds) * time.Second)
|
||||
defer timer.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case job := <-w.jobChan:
|
||||
case <-w.wakeChan:
|
||||
w.updateActivity()
|
||||
w.processJobs(ctx, &job)
|
||||
w.processJobs(ctx)
|
||||
|
||||
case <-timer.C:
|
||||
// Timer expired - fetch jobs from database
|
||||
if w.timerSeconds > 0 {
|
||||
w.updateActivity()
|
||||
w.processJobs(ctx, nil)
|
||||
w.processJobs(ctx)
|
||||
}
|
||||
timer.Reset(time.Duration(w.timerSeconds) * time.Second)
|
||||
|
||||
@@ -147,84 +193,101 @@ func (w *Worker) processLoop(ctx context.Context) {
|
||||
}
|
||||
|
||||
// processJobs processes jobs from the queue within a transaction
|
||||
func (w *Worker) processJobs(ctx context.Context, specificJob *models.Job) {
|
||||
func (w *Worker) processJobs(ctx context.Context) {
|
||||
defer w.recoverPanic()
|
||||
|
||||
for i := 0; i < w.fetchSize; i++ {
|
||||
|
||||
tx, err := w.db.Begin(ctx) // Start transaction
|
||||
tx, err := w.db.Begin(ctx)
|
||||
if err != nil {
|
||||
w.logger.Error("failed to begin transaction", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
var jobID int64
|
||||
if err := w.setTenantTx(ctx, tx); err != nil {
|
||||
tx.Rollback()
|
||||
w.logger.Error("failed to set tenant", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
if specificJob != nil && specificJob.ID > 0 {
|
||||
jobID = specificJob.ID
|
||||
specificJob = nil // Only process once
|
||||
} else {
|
||||
jobID, err = w.fetchNextJobTx(ctx, tx) // Use transaction
|
||||
if err != nil {
|
||||
tx.Rollback() // Rollback on fetch error
|
||||
w.logger.Error("failed to fetch job", "error", err)
|
||||
return
|
||||
}
|
||||
jobID, leaseToken, err := w.fetchNextJobTx(ctx, tx)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
w.logger.Error("failed to fetch job", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
if jobID <= 0 {
|
||||
tx.Rollback() // No job found, rollback
|
||||
return // No more jobs
|
||||
return // No more jobs
|
||||
}
|
||||
|
||||
// Run the job
|
||||
if err := w.runJobTx(ctx, tx, jobID); err != nil { // Use transaction
|
||||
tx.Rollback() // Rollback on job execution error
|
||||
if err := w.runJobTx(ctx, tx, jobID, leaseToken); err != nil {
|
||||
tx.Rollback() // Rollback on genuine infra failure
|
||||
w.logger.Error("failed to run job", "job_id", jobID, "error", err)
|
||||
} else {
|
||||
tx.Commit() // Commit if job successful
|
||||
if err := tx.Commit(); err != nil {
|
||||
w.logger.Error("failed to commit job", "job_id", jobID, "error", err)
|
||||
continue
|
||||
}
|
||||
w.jobsHandled++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fetchNextJobTx fetches the next job from the queue within a transaction
|
||||
func (w *Worker) fetchNextJobTx(ctx context.Context, tx adapter.DBTransaction) (int64, error) {
|
||||
// setTenantTx applies this worker's RLS tenant for the duration of tx.
|
||||
func (w *Worker) setTenantTx(ctx context.Context, tx adapter.DBTransaction) error {
|
||||
tenantID := w.tenantID
|
||||
if tenantID == "" {
|
||||
tenantID = "default"
|
||||
}
|
||||
_, err := tx.Exec(ctx, "SELECT broker.broker_set_tenant($1)", tenantID)
|
||||
return err
|
||||
}
|
||||
|
||||
// fetchNextJobTx fetches the next job from the queue within a transaction,
|
||||
// claiming it with a lease that must be presented back to broker_run.
|
||||
func (w *Worker) fetchNextJobTx(ctx context.Context, tx adapter.DBTransaction) (int64, string, error) {
|
||||
var retval int
|
||||
var errmsg string
|
||||
var nullableJobID sql.NullInt64
|
||||
var nullableLeaseToken sql.NullString
|
||||
|
||||
err := tx.QueryRow(ctx,
|
||||
"SELECT p_retval, p_errmsg, p_job_id FROM broker_get($1, $2)",
|
||||
w.QueueNumber, w.InstanceID,
|
||||
).Scan(&retval, &errmsg, &nullableJobID)
|
||||
"SELECT p_retval, p_errmsg, p_job_id, p_lease_token FROM broker.broker_get($1, $2, $3)",
|
||||
w.QueueNumber, w.InstanceID, w.leaseSeconds,
|
||||
).Scan(&retval, &errmsg, &nullableJobID, &nullableLeaseToken)
|
||||
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("query error: %w", err)
|
||||
return 0, "", fmt.Errorf("query error: %w", err)
|
||||
}
|
||||
|
||||
if retval > 0 {
|
||||
return 0, fmt.Errorf("broker_get error: %s", errmsg)
|
||||
return 0, "", fmt.Errorf("broker_get error: %s", errmsg)
|
||||
}
|
||||
|
||||
if !nullableJobID.Valid {
|
||||
return 0, nil
|
||||
return 0, "", nil
|
||||
}
|
||||
|
||||
return nullableJobID.Int64, nil
|
||||
return nullableJobID.Int64, nullableLeaseToken.String, nil
|
||||
}
|
||||
|
||||
// runJobTx executes a job within a transaction
|
||||
func (w *Worker) runJobTx(ctx context.Context, tx adapter.DBTransaction, jobID int64) error {
|
||||
// runJobTx executes a leased job within a transaction. It only returns an
|
||||
// error (triggering a rollback of the claim) on a genuine infra failure --
|
||||
// job outcomes reported via p_job_status (requeued/completed/dead-lettered)
|
||||
// are always committed.
|
||||
func (w *Worker) runJobTx(ctx context.Context, tx adapter.DBTransaction, jobID int64, leaseToken string) error {
|
||||
w.logger.Debug("running job", "job_id", jobID)
|
||||
|
||||
var retval int
|
||||
var errmsg string
|
||||
var jobStatus int
|
||||
|
||||
err := tx.QueryRow(ctx,
|
||||
"SELECT p_retval, p_errmsg FROM broker_run($1)",
|
||||
jobID,
|
||||
).Scan(&retval, &errmsg)
|
||||
"SELECT p_retval, p_errmsg, p_job_status FROM broker.broker_run($1, $2)",
|
||||
jobID, leaseToken,
|
||||
).Scan(&retval, &errmsg, &jobStatus)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("query error: %w", err)
|
||||
@@ -234,7 +297,7 @@ func (w *Worker) runJobTx(ctx context.Context, tx adapter.DBTransaction, jobID i
|
||||
return fmt.Errorf("broker_run error: %s", errmsg)
|
||||
}
|
||||
|
||||
w.logger.Debug("job completed", "job_id", jobID)
|
||||
w.logger.Debug("job finished", "job_id", jobID, "job_status", jobStatus)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -255,6 +318,6 @@ func (w *Worker) GetStats() (lastActivity time.Time, jobsHandled int64, running
|
||||
// recoverPanic recovers from panics in the worker
|
||||
func (w *Worker) recoverPanic() {
|
||||
if r := recover(); r != nil {
|
||||
w.logger.Error("worker panic recovered", "panic", r)
|
||||
w.logger.Error("worker panic recovered", "panic", r, "stack", string(debug.Stack()))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user