From 1214f69e0c16daf31a93d7f35b772e0b5d81fb45 Mon Sep 17 00:00:00 2001 From: Hein Date: Wed, 30 Sep 2026 16:16:25 +0200 Subject: [PATCH] feat(middleware): export clientqueue_enqueued_total counter Counts every request placed in a wait queue regardless of outcome, so the total ever queued no longer has to be summed from queued, timeout and canceled. --- pkg/middleware/README.md | 1 + pkg/middleware/clientqueue.go | 6 ++++++ pkg/middleware/clientqueue_test.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/pkg/middleware/README.md b/pkg/middleware/README.md index 01b7811..59f504e 100644 --- a/pkg/middleware/README.md +++ b/pkg/middleware/README.md @@ -449,6 +449,7 @@ no per-client labels so cardinality stays bounded. | Metric | Type | Meaning | |---|---|---| | `clientqueue_requests_total{result}` | counter | `immediate`, `queued`, `rejected_full`, `timeout`, `canceled` | +| `clientqueue_enqueued_total` | counter | requests ever placed in a wait queue, whatever happened next (ran, timed out, cancelled) | | `clientqueue_wait_seconds` | histogram | wait for a slot; 0 for requests that ran immediately | | `clientqueue_wait_max_seconds` | gauge | longest wait since process start | | `clientqueue_burst_size` | histogram | peak outstanding requests per client busy period | diff --git a/pkg/middleware/clientqueue.go b/pkg/middleware/clientqueue.go index bc647e2..08e9783 100644 --- a/pkg/middleware/clientqueue.go +++ b/pkg/middleware/clientqueue.go @@ -70,6 +70,11 @@ var ( Help: "Requests currently waiting for a slot", }) + queueEnqueued = promauto.NewCounter(prometheus.CounterOpts{ + Name: "clientqueue_enqueued_total", + Help: "Requests ever placed in a wait queue, whatever their outcome (ran, timed out or cancelled)", + }) + queueWaitingClients = promauto.NewGauge(prometheus.GaugeOpts{ Name: "clientqueue_waiting_clients", Help: "Clients currently with at least one request waiting for a slot", @@ -286,6 +291,7 @@ func (q *ClientQueue) acquire(ctx context.Context, key string) error { c.enter() q.mu.Unlock() queueDepth.Inc() + queueEnqueued.Inc() timer := time.NewTimer(q.cfg.MaxWait) defer timer.Stop() diff --git a/pkg/middleware/clientqueue_test.go b/pkg/middleware/clientqueue_test.go index 53f59aa..9fd5c58 100644 --- a/pkg/middleware/clientqueue_test.go +++ b/pkg/middleware/clientqueue_test.go @@ -279,6 +279,7 @@ func TestClientQueueMetrics(t *testing.T) { q := newTestQueue(t, ClientQueueConfig{MaxConcurrent: 2}) imm0 := gaugeVal(t, queueRequests.WithLabelValues("immediate")) que0 := gaugeVal(t, queueRequests.WithLabelValues("queued")) + enq0 := gaugeVal(t, queueEnqueued) bc0, bs0 := histVal(t, queueBurst) wc0, _ := histVal(t, queueWait) @@ -318,6 +319,9 @@ func TestClientQueueMetrics(t *testing.T) { if got := gaugeVal(t, queueRequests.WithLabelValues("immediate")) - imm0; got != 2 { t.Errorf("immediate = %v, want 2", got) } + if got := gaugeVal(t, queueEnqueued) - enq0; got != 3 { + t.Errorf("enqueued = %v, want 3", got) + } if got := gaugeVal(t, queueRequests.WithLabelValues("queued")) - que0; got != 3 { t.Errorf("queued = %v, want 3", got) } @@ -404,3 +408,27 @@ func TestClientQueueWaitingClientsGauge(t *testing.T) { waitFor(0) q.release("b") } + +func TestClientQueueEnqueuedCountsAllOutcomes(t *testing.T) { + q := newTestQueue(t, ClientQueueConfig{MaxConcurrent: 1, MaxQueue: 1, MaxWait: 50 * time.Millisecond}) + base := gaugeVal(t, queueEnqueued) + if err := q.acquire(t.Context(), "k"); err != nil { // immediate: not enqueued + t.Fatal(err) + } + if err := q.acquire(t.Context(), "k"); !errors.Is(err, errQueueWait) { // enqueued, times out + t.Fatalf("err = %v, want timeout", err) + } + cctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- q.acquire(cctx, "k") }() // enqueued, cancelled + time.Sleep(10 * time.Millisecond) + if err := q.acquire(t.Context(), "k"); !errors.Is(err, errQueueFull) { // rejected: not enqueued + t.Fatalf("err = %v, want queue full", err) + } + cancel() + <-done + q.release("k") + if got := gaugeVal(t, queueEnqueued) - base; got != 2 { + t.Fatalf("enqueued = %v, want 2 (timeout + cancel; not immediate or rejected)", got) + } +}