feat: add prometheus metrics and dashboard

pkg/broker/worker/worker.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
+	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/metrics"
 )
 
 // Worker represents a single job processing worker
@@ -29,6 +30,8 @@ type Worker struct {
 	fetchSize    int
 	tenantID     string
 	leaseSeconds int
+	metrics      *metrics.Metrics
+	databaseName string
 }
 
 // Stats holds worker statistics
@@ -50,6 +53,8 @@ type Config struct {
 	FetchSize    int
 	TenantID     string
 	LeaseSeconds int
+	Metrics      *metrics.Metrics
+	DatabaseName string
 }
 
 // New creates a new worker
@@ -72,6 +77,8 @@ func New(cfg Config) *Worker {
 		fetchSize:    cfg.FetchSize,
 		tenantID:     cfg.TenantID,
 		leaseSeconds: leaseSeconds,
+		metrics:      cfg.Metrics,
+		databaseName: cfg.DatabaseName,
 	}
 }
 
@@ -228,8 +235,19 @@ func (w *Worker) processJobs(ctx context.Context) {
 			return // No more jobs
 		}
 
+		jobName, jobGroup, err := w.fetchJobLabelsTx(ctx, tx, jobID)
+		if err != nil {
+			w.logger.Warn("failed to fetch job labels for metrics", "job_id", jobID, "error", err)
+		}
+
 		// Run the job
-		if err := w.runJobTx(ctx, tx, jobID, leaseToken); err != nil {
+		start := time.Now()
+		jobStatus, err := w.runJobTx(ctx, tx, jobID, leaseToken)
+		duration := time.Since(start)
+		if err == nil {
+			w.recordJobMetric(jobStatus, jobGroup, jobName, duration)
+		}
+		if err != nil {
 			// Rollback on genuine infra failure
 			if rbErr := tx.Rollback(); rbErr != nil {
 				w.logger.Error("failed to rollback transaction", "error", rbErr)
@@ -287,7 +305,7 @@ func (w *Worker) fetchNextJobTx(ctx context.Context, tx adapter.DBTransaction) (
 // error (triggering a rollback of the claim) on a genuine infra failure --
 // job outcomes reported via p_job_status (requeued/completed/dead-lettered)
 // are always committed.
-func (w *Worker) runJobTx(ctx context.Context, tx adapter.DBTransaction, jobID int64, leaseToken string) error {
+func (w *Worker) runJobTx(ctx context.Context, tx adapter.DBTransaction, jobID int64, leaseToken string) (int, error) {
 	w.logger.Debug("running job", "job_id", jobID)
 
 	var retval int
@@ -300,15 +318,43 @@ func (w *Worker) runJobTx(ctx context.Context, tx adapter.DBTransaction, jobID i
 	).Scan(&retval, &errmsg, &jobStatus)
 
 	if err != nil {
-		return fmt.Errorf("query error: %w", err)
+		return 0, fmt.Errorf("query error: %w", err)
 	}
 
 	if retval > 0 {
-		return fmt.Errorf("broker_run error: %s", errmsg)
+		return 0, fmt.Errorf("broker_run error: %s", errmsg)
 	}
 
 	w.logger.Debug("job finished", "job_id", jobID, "job_status", jobStatus)
-	return nil
+	return jobStatus, nil
+}
+
+// fetchJobLabelsTx looks up the job_name/job_group of jobID for metric
+// labeling. Best-effort: callers log and continue on error rather than
+// failing the job over a metrics lookup.
+func (w *Worker) fetchJobLabelsTx(ctx context.Context, tx adapter.DBTransaction, jobID int64) (jobName, jobGroup string, err error) {
+	err = tx.QueryRow(ctx,
+		"SELECT job_name, job_group FROM broker.broker_jobs WHERE id_broker_jobs = $1",
+		jobID,
+	).Scan(&jobName, &jobGroup)
+	if err != nil {
+		return "", "", fmt.Errorf("query error: %w", err)
+	}
+	return jobName, jobGroup, nil
+}
+
+// recordJobMetric routes a finished job attempt to the appropriate
+// Prometheus counter/histogram based on the p_job_status broker_run
+// reported (0=requeued, 2=completed, 3=dead-lettered).
+func (w *Worker) recordJobMetric(jobStatus int, jobGroup, jobName string, duration time.Duration) {
+	switch jobStatus {
+	case 2:
+		w.metrics.RecordJobCompleted(w.databaseName, jobGroup, jobName, duration)
+	case 3:
+		w.metrics.RecordJobFailed(w.databaseName, jobGroup, jobName, duration)
+	case 0:
+		w.metrics.RecordJobRequeued(w.databaseName, jobGroup, jobName, duration)
+	}
 }
 
 // updateActivity updates the last activity timestamp