wdevs/pgsql-broker
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

feat: add prometheus metrics and dashboard
Integration Tests / integration-test (pull_request) Successful in 2m33s
Details

Browse Source
This commit is contained in:
SG Command
2026-09-19 05:12:04 +02:00
parent d3bce39783
commit f69cbea49f
12 changed files with 613 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/broker/metrics/metrics.go
+130
View File
@@ -0,0 +1,130 @@
// Package metrics defines the broker's Prometheus collectors and a small
// embedded HTTP server that exposes them, both as the standard /metrics
// exposition endpoint and as a single-page HTML dashboard.
package metrics
import (
"strconv"
"time"
"github.com/prometheus/client_golang/prometheus"
)
// Metrics holds every Prometheus collector the broker exposes. It is safe
// for concurrent use, and a nil *Metrics is safe to call methods on (all
// recording methods become no-ops), so callers that construct a Broker
// without metrics enabled don't need to special-case it.
type Metrics struct {
Registry *prometheus.Registry
jobsCompleted *prometheus.CounterVec
jobsFailed *prometheus.CounterVec
jobsRequeued *prometheus.CounterVec
jobDuration *prometheus.HistogramVec
jobsQueued *prometheus.GaugeVec
databaseCount prometheus.Gauge
queueCount *prometheus.GaugeVec
}
// New creates a Metrics instance with a fresh (non-global) registry, so
// multiple brokers can coexist in the same process -- e.g. in tests --
// without colliding on prometheus.DefaultRegisterer.
func New() *Metrics {
registry := prometheus.NewRegistry()
m := &Metrics{
Registry: registry,
jobsCompleted: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "broker_jobs_completed_total",
Help: "Total number of jobs that completed successfully.",
}, []string{"database", "job_group", "job_name"}),
jobsFailed: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "broker_jobs_failed_total",
Help: "Total number of jobs that were dead-lettered after exhausting retries.",
}, []string{"database", "job_group", "job_name"}),
jobsRequeued: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "broker_jobs_requeued_total",
Help: "Total number of job attempts that failed and were requeued for retry.",
}, []string{"database", "job_group", "job_name"}),
jobDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "broker_job_duration_seconds",
Help: "Job execution duration in seconds, by group and name.",
Buckets: prometheus.DefBuckets,
}, []string{"database", "job_group", "job_name"}),
jobsQueued: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "broker_jobs_queued",
Help: "Current number of pending (not yet claimed) jobs, by database and queue.",
}, []string{"database", "queue"}),
databaseCount: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "broker_databases",
Help: "Number of database instances managed by this broker process.",
}),
queueCount: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "broker_queues",
Help: "Number of queues configured for a database instance.",
}, []string{"database"}),
}
registry.MustRegister(
m.jobsCompleted,
m.jobsFailed,
m.jobsRequeued,
m.jobDuration,
m.jobsQueued,
m.databaseCount,
m.queueCount,
)
return m
}
// RecordJobCompleted records a successfully completed job attempt.
func (m *Metrics) RecordJobCompleted(database, group, name string, duration time.Duration) {
if m == nil {
return
}
m.jobsCompleted.WithLabelValues(database, group, name).Inc()
m.jobDuration.WithLabelValues(database, group, name).Observe(duration.Seconds())
}
// RecordJobFailed records a job attempt that was dead-lettered (attempts exhausted).
func (m *Metrics) RecordJobFailed(database, group, name string, duration time.Duration) {
if m == nil {
return
}
m.jobsFailed.WithLabelValues(database, group, name).Inc()
m.jobDuration.WithLabelValues(database, group, name).Observe(duration.Seconds())
}
// RecordJobRequeued records a job attempt that failed but was requeued for retry.
func (m *Metrics) RecordJobRequeued(database, group, name string, duration time.Duration) {
if m == nil {
return
}
m.jobsRequeued.WithLabelValues(database, group, name).Inc()
m.jobDuration.WithLabelValues(database, group, name).Observe(duration.Seconds())
}
// SetJobsQueued sets the current pending job count for a database/queue pair.
func (m *Metrics) SetJobsQueued(database string, queue int, count float64) {
if m == nil {
return
}
m.jobsQueued.WithLabelValues(database, strconv.Itoa(queue)).Set(count)
}
// SetDatabaseCount sets the number of database instances managed by this process.
func (m *Metrics) SetDatabaseCount(n int) {
if m == nil {
return
}
m.databaseCount.Set(float64(n))
}
// SetQueueCount sets the number of queues configured for a database instance.
func (m *Metrics) SetQueueCount(database string, n int) {
if m == nil {
return
}
m.queueCount.WithLabelValues(database).Set(float64(n))
}
pkg/broker/metrics/page.html
+215
View File
@@ -0,0 +1,215 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>pgsql-broker metrics</title>
<style>
:root {
color-scheme: dark;
--bg: #0f1115;
--panel: #161a22;
--border: #262b36;
--text: #e6e9ef;
--muted: #8b93a7;
--accent: #5aa8ff;
--good: #4caf7d;
--bad: #e5636b;
}
* { box-sizing: border-box; }
body {
margin: 0;
padding: 2rem;
background: var(--bg);
color: var(--text);
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
}
h1 {
font-size: 1.25rem;
margin: 0 0 0.25rem;
}
#status {
color: var(--muted);
font-size: 0.8rem;
margin-bottom: 1.5rem;
}
.grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(360px, 1fr));
gap: 1rem;
}
.card {
background: var(--panel);
border: 1px solid var(--border);
border-radius: 8px;
padding: 1rem;
}
.card h2 {
font-size: 0.9rem;
margin: 0 0 0.75rem;
color: var(--accent);
font-weight: 600;
}
.card .help {
color: var(--muted);
font-size: 0.75rem;
margin: -0.5rem 0 0.75rem;
}
table {
width: 100%;
border-collapse: collapse;
font-size: 0.8rem;
}
td, th {
text-align: left;
padding: 0.2rem 0.4rem 0.2rem 0;
border-bottom: 1px solid var(--border);
}
td.value {
text-align: right;
font-variant-numeric: tabular-nums;
white-space: nowrap;
}
td.labels {
color: var(--muted);
}
.completed .value { color: var(--good); }
.failed .value { color: var(--bad); }
.empty {
color: var(--muted);
font-size: 0.8rem;
}
</style>
</head>
<body>
<h1>pgsql-broker metrics</h1>
<div id="status">loading&hellip;</div>
<div class="grid" id="grid"></div>
<script>
(function () {
"use strict";
// Metric name -> { help, className } for the cards we render, in order.
var METRICS = [
{ name: "broker_jobs_completed_total", title: "Jobs completed", cls: "completed" },
{ name: "broker_jobs_failed_total", title: "Jobs failed (dead-lettered)", cls: "failed" },
{ name: "broker_jobs_requeued_total", title: "Jobs requeued (retries)", cls: "" },
{ name: "broker_jobs_queued", title: "Jobs currently queued", cls: "" },
{ name: "broker_job_duration_seconds_sum", title: "Job duration, total seconds by group/name", cls: "" },
{ name: "broker_job_duration_seconds_count", title: "Job duration, sample count by group/name", cls: "" },
{ name: "broker_databases", title: "Databases managed", cls: "" },
{ name: "broker_queues", title: "Queues per database", cls: "" }
];
// Parses Prometheus text exposition format into { name: [{labels, value}] }.
function parseMetrics(text) {
var byName = {};
var lines = text.split("\n");
for (var i = 0; i < lines.length; i++) {
var line = lines[i];
if (!line || line[0] === "#") continue;
var name, labels = {}, rest;
var braceIdx = line.indexOf("{");
var spaceIdx;
if (braceIdx !== -1) {
name = line.slice(0, braceIdx);
var closeIdx = line.indexOf("}", braceIdx);
if (closeIdx === -1) continue;
var labelStr = line.slice(braceIdx + 1, closeIdx);
var labelRe = /(\w+)="((?:[^"\\]|\\.)*)"/g;
var m;
while ((m = labelRe.exec(labelStr)) !== null) {
labels[m[1]] = m[2].replace(/\\"/g, '"').replace(/\\\\/g, "\\");
}
rest = line.slice(closeIdx + 1).trim();
} else {
spaceIdx = line.indexOf(" ");
if (spaceIdx === -1) continue;
name = line.slice(0, spaceIdx);
rest = line.slice(spaceIdx + 1).trim();
}
var value = parseFloat(rest.split(" ")[0]);
if (isNaN(value)) continue;
if (!byName[name]) byName[name] = [];
byName[name].push({ labels: labels, value: value });
}
return byName;
}
function formatLabels(labels) {
var keys = Object.keys(labels).sort();
return keys.map(function (k) { return k + "=" + labels[k]; }).join(", ");
}
function formatValue(v) {
if (Number.isInteger(v)) return String(v);
return v.toFixed(3);
}
function render(byName) {
var grid = document.getElementById("grid");
grid.innerHTML = "";
METRICS.forEach(function (spec) {
var series = byName[spec.name] || [];
var card = document.createElement("div");
card.className = "card " + spec.cls;
var h2 = document.createElement("h2");
h2.textContent = spec.title;
card.appendChild(h2);
if (series.length === 0) {
var empty = document.createElement("div");
empty.className = "empty";
empty.textContent = "no data yet";
card.appendChild(empty);
} else {
var table = document.createElement("table");
series.sort(function (a, b) { return b.value - a.value; });
series.forEach(function (s) {
var tr = document.createElement("tr");
var tdLabels = document.createElement("td");
tdLabels.className = "labels";
tdLabels.textContent = formatLabels(s.labels) || "(none)";
var tdValue = document.createElement("td");
tdValue.className = "value";
tdValue.textContent = formatValue(s.value);
tr.appendChild(tdLabels);
tr.appendChild(tdValue);
table.appendChild(tr);
});
card.appendChild(table);
}
grid.appendChild(card);
});
}
function refresh() {
fetch("metrics", { cache: "no-store" })
.then(function (resp) {
if (!resp.ok) throw new Error("HTTP " + resp.status);
return resp.text();
})
.then(function (text) {
render(parseMetrics(text));
document.getElementById("status").textContent =
"last updated " + new Date().toLocaleTimeString();
})
.catch(function (err) {
document.getElementById("status").textContent =
"failed to load metrics: " + err.message;
});
}
refresh();
setInterval(refresh, 5000);
})();
</script>
</body>
</html>
pkg/broker/metrics/server.go
+74
View File
@@ -0,0 +1,74 @@
package metrics
import (
"context"
_ "embed"
"fmt"
"net"
"net/http"
"github.com/prometheus/client_golang/prometheus/promhttp"
"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
)
//go:embed page.html
var dashboardHTML []byte
// Server is the embedded HTTP server exposing /metrics (standard Prometheus
// exposition format) and / (a single-page HTML dashboard that polls
// /metrics).
type Server struct {
httpServer *http.Server
listener net.Listener
logger adapter.Logger
}
// NewServer builds a Server bound to addr (e.g. "127.0.0.1:9469"). Binding
// happens immediately so a port conflict is reported to the caller rather
// than surfacing later in a background goroutine.
func NewServer(m *Metrics, addr string, logger adapter.Logger) (*Server, error) {
listener, err := net.Listen("tcp", addr)
if err != nil {
return nil, fmt.Errorf("failed to bind metrics server to %s: %w", addr, err)
}
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.HandlerFor(m.Registry, promhttp.HandlerOpts{}))
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write(dashboardHTML)
})
return &Server{
httpServer: &http.Server{Handler: mux},
listener: listener,
logger: logger.With("component", "metrics-server"),
}, nil
}
// Addr returns the actual bound address (useful when addr was given with a
// ":0" port).
func (s *Server) Addr() string {
return s.listener.Addr().String()
}
// Start serves in the background. It returns immediately; Serve errors
// (other than a clean Shutdown) are logged.
func (s *Server) Start() {
s.logger.Info("metrics server listening", "addr", s.Addr())
go func() {
if err := s.httpServer.Serve(s.listener); err != nil && err != http.ErrServerClosed {
s.logger.Error("metrics server stopped unexpectedly", "error", err)
}
}()
}
// Stop gracefully shuts down the server.
func (s *Server) Stop(ctx context.Context) error {
return s.httpServer.Shutdown(ctx)
}