feat: add prometheus metrics and dashboard

2026-09-19 05:12:04 +02:00
parent d3bce39783
commit f69cbea49f
12 changed files with 613 additions and 13 deletions
@@ -0,0 +1,130 @@
+// Package metrics defines the broker's Prometheus collectors and a small
+// embedded HTTP server that exposes them, both as the standard /metrics
+// exposition endpoint and as a single-page HTML dashboard.
+package metrics
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metrics holds every Prometheus collector the broker exposes. It is safe
+// for concurrent use, and a nil *Metrics is safe to call methods on (all
+// recording methods become no-ops), so callers that construct a Broker
+// without metrics enabled don't need to special-case it.
+type Metrics struct {
+	Registry *prometheus.Registry
+
+	jobsCompleted *prometheus.CounterVec
+	jobsFailed    *prometheus.CounterVec
+	jobsRequeued  *prometheus.CounterVec
+	jobDuration   *prometheus.HistogramVec
+	jobsQueued    *prometheus.GaugeVec
+	databaseCount prometheus.Gauge
+	queueCount    *prometheus.GaugeVec
+}
+
+// New creates a Metrics instance with a fresh (non-global) registry, so
+// multiple brokers can coexist in the same process -- e.g. in tests --
+// without colliding on prometheus.DefaultRegisterer.
+func New() *Metrics {
+	registry := prometheus.NewRegistry()
+
+	m := &Metrics{
+		Registry: registry,
+		jobsCompleted: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "broker_jobs_completed_total",
+			Help: "Total number of jobs that completed successfully.",
+		}, []string{"database", "job_group", "job_name"}),
+		jobsFailed: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "broker_jobs_failed_total",
+			Help: "Total number of jobs that were dead-lettered after exhausting retries.",
+		}, []string{"database", "job_group", "job_name"}),
+		jobsRequeued: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "broker_jobs_requeued_total",
+			Help: "Total number of job attempts that failed and were requeued for retry.",
+		}, []string{"database", "job_group", "job_name"}),
+		jobDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "broker_job_duration_seconds",
+			Help:    "Job execution duration in seconds, by group and name.",
+			Buckets: prometheus.DefBuckets,
+		}, []string{"database", "job_group", "job_name"}),
+		jobsQueued: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "broker_jobs_queued",
+			Help: "Current number of pending (not yet claimed) jobs, by database and queue.",
+		}, []string{"database", "queue"}),
+		databaseCount: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "broker_databases",
+			Help: "Number of database instances managed by this broker process.",
+		}),
+		queueCount: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "broker_queues",
+			Help: "Number of queues configured for a database instance.",
+		}, []string{"database"}),
+	}
+
+	registry.MustRegister(
+		m.jobsCompleted,
+		m.jobsFailed,
+		m.jobsRequeued,
+		m.jobDuration,
+		m.jobsQueued,
+		m.databaseCount,
+		m.queueCount,
+	)
+
+	return m
+}
+
+// RecordJobCompleted records a successfully completed job attempt.
+func (m *Metrics) RecordJobCompleted(database, group, name string, duration time.Duration) {
+	if m == nil {
+		return
+	}
+	m.jobsCompleted.WithLabelValues(database, group, name).Inc()
+	m.jobDuration.WithLabelValues(database, group, name).Observe(duration.Seconds())
+}
+
+// RecordJobFailed records a job attempt that was dead-lettered (attempts exhausted).
+func (m *Metrics) RecordJobFailed(database, group, name string, duration time.Duration) {
+	if m == nil {
+		return
+	}
+	m.jobsFailed.WithLabelValues(database, group, name).Inc()
+	m.jobDuration.WithLabelValues(database, group, name).Observe(duration.Seconds())
+}
+
+// RecordJobRequeued records a job attempt that failed but was requeued for retry.
+func (m *Metrics) RecordJobRequeued(database, group, name string, duration time.Duration) {
+	if m == nil {
+		return
+	}
+	m.jobsRequeued.WithLabelValues(database, group, name).Inc()
+	m.jobDuration.WithLabelValues(database, group, name).Observe(duration.Seconds())
+}
+
+// SetJobsQueued sets the current pending job count for a database/queue pair.
+func (m *Metrics) SetJobsQueued(database string, queue int, count float64) {
+	if m == nil {
+		return
+	}
+	m.jobsQueued.WithLabelValues(database, strconv.Itoa(queue)).Set(count)
+}
+
+// SetDatabaseCount sets the number of database instances managed by this process.
+func (m *Metrics) SetDatabaseCount(n int) {
+	if m == nil {
+		return
+	}
+	m.databaseCount.Set(float64(n))
+}
+
+// SetQueueCount sets the number of queues configured for a database instance.
+func (m *Metrics) SetQueueCount(database string, n int) {
+	if m == nil {
+		return
+	}
+	m.queueCount.WithLabelValues(database).Set(float64(n))
+}