Merge pull request 'feat(cli): manage configured databases' (#5) from issue-3-cli-database-management into main

Reviewed-on: #5
Reviewed-by: Warky <2+warkanum@noreply@warky.dev>

.dockerignore

@@ -0,0 +1,6 @@
+.git
+bin
+broker.log
+broker.pid
+plan
+*.md

.env.example

@@ -0,0 +1,11 @@
+# Copy to .env (gitignored) and fill in real values.
+# POSTGRES_PASSWORD is the postgres superuser password (used only by the
+# one-shot `migrate` service to run `install --with-roles`).
+# The three BROKER_*_PASSWORD values must match the passwords you put in
+# broker.docker.yaml (copied from broker.docker.example.yaml) for the
+# corresponding role -- BROKER_RUNTIME_PASSWORD in particular must match
+# the `password:` field the broker service itself connects with.
+POSTGRES_PASSWORD=change_me
+BROKER_ADMIN_PASSWORD=change_me
+BROKER_RUNTIME_PASSWORD=change_me
+BROKER_ENQUEUE_PASSWORD=change_me

.github/workflows/docker-release.yml

@@ -0,0 +1,74 @@
+name: Build & Release Docker Image
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Existing tag to release (e.g. v0.1.0)'
+        required: true
+        type: string
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    env:
+      IMAGE: git.warky.dev/wdevs/pgsql-broker
+      TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
+    steps:
+      - name: Validate release tag
+        run: |
+          case "$TAG" in
+            v*) ;;
+            *) echo "Release tags must start with v (received: $TAG)" >&2; exit 1 ;;
+          esac
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref }}
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Verify package registry credentials
+        env:
+          PACKAGE_REGISTRY_USERNAME: ${{ secrets.PACKAGE_REGISTRY_USERNAME }}
+          PACKAGE_REGISTRY_TOKEN: ${{ secrets.PACKAGE_REGISTRY_TOKEN }}
+        run: |
+          test -n "$PACKAGE_REGISTRY_USERNAME" || {
+            echo 'PACKAGE_REGISTRY_USERNAME is required to publish the Docker image.' >&2
+            exit 1
+          }
+          test -n "$PACKAGE_REGISTRY_TOKEN" || {
+            echo 'PACKAGE_REGISTRY_TOKEN is required to publish the Docker image.' >&2
+            exit 1
+          }
+
+      - name: Log in to the Warky container registry
+        uses: docker/login-action@v3
+        with:
+          registry: git.warky.dev
+          username: ${{ secrets.PACKAGE_REGISTRY_USERNAME }}
+          password: ${{ secrets.PACKAGE_REGISTRY_TOKEN }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: Dockerfile
+          push: true
+          build-args: |
+            VERSION=${{ env.TAG }}
+            COMMIT=${{ github.sha }}
+            BUILD_TIME=${{ github.event.head_commit.timestamp || github.event.repository.updated_at }}
+          tags: |
+            ${{ env.IMAGE }}:${{ env.TAG }}
+            ${{ env.IMAGE }}:latest
+          labels: |
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.version=${{ env.TAG }}

.github/workflows/integration.yml

@@ -12,6 +12,19 @@ jobs:
   integration-test:
     runs-on: ubuntu-latest
 
+    services:
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_DB: broker_test
+          POSTGRES_USER: user
+          POSTGRES_PASSWORD: password
+        options: >-
+          --health-cmd="pg_isready -U user"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=10
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -19,16 +32,28 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.25'
+          go-version: '1.26'
           cache: true
 
-      - name: Set up Python
+      - name: Check formatting
-        uses: actions/setup-python@v5
+        run: |
-        with:
+          make fmt
-          python-version: '3.12'
+          git diff --exit-code -- '*.go'
 
-      - name: Install podman-compose
+      - name: Install golangci-lint
-        run: pip install podman-compose
+        run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
+
+      - name: Run lint
+        run: make lint
 
       - name: Run all tests
-        run: make test-all
+        env:
+          # act_runner runs this job in its own container alongside the
+          # postgres service container, both on the job's Docker network.
+          # "localhost" from inside the job container is the job container
+          # itself, not the runner host, so the service must be reached by
+          # its network alias (the services: key) and container-internal
+          # port -- not a published host port.
+          TEST_DB_HOST: postgres
+          TEST_DB_PORT: 5432
+        run: make test-ci TEST_DB_HOST="$TEST_DB_HOST" TEST_DB_PORT="$TEST_DB_PORT"

.github/workflows/release.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         cache: true
 
     - name: Get version from tag

.gitignore

@@ -32,7 +32,9 @@ bin/
 broker.yaml
 broker.yml
 broker.json
+broker.docker.yaml
 !broker.example.yaml
+!broker.docker.example.yaml
 
 # IDE
 .vscode/

Dockerfile

@@ -0,0 +1,40 @@
+# syntax=docker/dockerfile:1
+
+# ---- Build stage ----
+FROM golang:1.26-alpine AS builder
+
+WORKDIR /src
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+ARG VERSION=dev
+ARG COMMIT=unknown
+ARG BUILD_TIME=unknown
+
+RUN CGO_ENABLED=0 GOOS=linux go build \
+      -trimpath \
+      -ldflags "-w -s -X main.Version=${VERSION} -X main.Commit=${COMMIT} -X main.BuildTime=${BUILD_TIME}" \
+      -o /out/pgsql-broker \
+      ./cmd/broker
+
+# ---- Runtime stage ----
+FROM alpine:3.22
+
+RUN apk add --no-cache ca-certificates tzdata && \
+    addgroup -S broker && adduser -S -G broker -h /home/broker broker && \
+    mkdir -p /etc/pgsql-broker && \
+    chown -R broker:broker /etc/pgsql-broker
+
+COPY --from=builder /out/pgsql-broker /usr/local/bin/pgsql-broker
+
+USER broker
+WORKDIR /home/broker
+
+# broker.yaml is expected to be mounted at /etc/pgsql-broker/broker.yaml
+# (config.LoadConfig's default search path) -- no config is baked into the
+# image, since it holds database credentials.
+ENTRYPOINT ["pgsql-broker"]
+CMD ["start"]

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all build clean test test-all test-integration-go test-unit-go test-connection schema-install broker-start broker-stop install deps docker-up docker-down help
+.PHONY: all build clean test test-all test-ci test-integration-go test-unit-go test-connection generate-test-config schema-install broker-start broker-stop install deps docker-up docker-down docker-build release help
 
 # Build variables
 BINARY_NAME=pgsql-broker
@@ -30,9 +30,16 @@ COMPOSE_CMD := $(shell \
 
 
 
+# Test database connection info. Override in CI to point at a dynamically
+# assigned Postgres (e.g. a services: block port), avoiding a fixed host port
+# that can collide with other jobs on a shared runner.
+TEST_DB_HOST ?= 127.0.0.1
+TEST_DB_PORT ?= 5433
+TEST_CONFIG := $(BIN_DIR)/broker.test.runtime.yaml
+
 # Version information
 VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
-BUILD_TIME=$(shell date -u '+2026-01-02_19:58:30')
+BUILD_TIME=$(shell date -u '+%Y-%m-%d_%H:%M:%S')
 COMMIT=$(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
 
 # Inject version info
@@ -63,28 +70,52 @@ test-local-unit: deps ## Run local unit tests
 	@echo "Running local unit tests..."
 	@$(GO) test -v -race -cover $(shell $(GO) list ./... | grep -v /tests/integration)
 
-test-all: test-teardown test-setup test-connection schema-install broker-start test-local-unit test-integration-go broker-stop test-teardown ## Run all unit and integration tests
+test-all: test-teardown test-setup test-connection schema-install test-local-unit test-integration-go test-teardown ## Run all unit and integration tests (starts its own Postgres via docker-compose)
+
+test-ci: test-connection schema-install test-local-unit test-integration-go ## Run all unit and integration tests against an externally-provided Postgres (CI services: block)
 
 test-connection: deps ## Test database connection with retry
-	@echo "Testing database connection..."
+	@echo "Testing database connection (host=$(TEST_DB_HOST) port=$(TEST_DB_PORT))..."
-	@$(GO) test -v ./tests/integration/connection_test.go
+	@TEST_DB_HOST=$(TEST_DB_HOST) TEST_DB_PORT=$(TEST_DB_PORT) $(GO) test -v -run '^TestConnection$$' ./tests/integration/...
 
-schema-install: build ## Install database schema using the broker CLI
+generate-test-config: ## (internal) render broker.test.yaml with TEST_DB_HOST/TEST_DB_PORT
+	@mkdir -p $(BIN_DIR)
+	@sed -e "s/^    host: .*/    host: $(TEST_DB_HOST)/" -e "s/^    port: .*/    port: $(TEST_DB_PORT)/" broker.test.yaml > $(TEST_CONFIG)
+
+schema-install: build generate-test-config ## Install database schema using the broker CLI
 	@echo "Installing database schema..."
-	@$(BIN_DIR)/$(BINARY_NAME) install --config broker.test.yaml
+	@$(BIN_DIR)/$(BINARY_NAME) install --config $(TEST_CONFIG)
 
-test-setup: build ## Start test environment (docker-compose)
+test-setup: build ## Start test environment (docker-compose/podman-compose)
-	@echo "Starting test environment..."
+	@echo "Starting test environment (using $(COMPOSE_CMD))..."
-	@podman-compose -f tests/docker-compose.yml up -d
+	@if [ "$(CONTAINER_RUNTIME)" = "none" ]; then \
+		echo "Error: Neither Docker nor Podman is installed"; \
+		exit 1; \
+	fi
+	@$(COMPOSE_CMD) -f tests/docker-compose.yml up -d
+	@echo "Waiting for PostgreSQL to be ready..."
+	@for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do \
+		if $(COMPOSE_CMD) -f tests/docker-compose.yml exec -T postgres pg_isready -U user > /dev/null 2>&1; then \
+			echo "PostgreSQL is ready"; exit 0; \
+		fi; \
+		sleep 2; \
+	done; \
+	echo "ERROR: PostgreSQL did not become ready in time"; \
+	$(COMPOSE_CMD) -f tests/docker-compose.yml logs postgres; \
+	exit 1
 
-test-teardown: ## Stop test environment (docker-compose)
+test-teardown: ## Stop test environment (docker-compose/podman-compose)
-	@echo "Stopping test environment..."
+	@echo "Stopping test environment (using $(COMPOSE_CMD))..."
-	@podman-compose -f tests/docker-compose.yml down -v --rmi all
+	@if [ "$(CONTAINER_RUNTIME)" = "none" ]; then \
-	@sleep 5 # Give Docker time to release resources
+		echo "Neither Docker nor Podman is installed, skipping teardown"; \
+	else \
+		$(COMPOSE_CMD) -f tests/docker-compose.yml down -v --rmi all || true; \
+	fi
+	@sleep 5 # Give the container runtime time to release resources
 
-broker-start: build ## Start the broker in the background
+broker-start: build generate-test-config ## Start the broker in the background
 	@echo "Starting broker..."
-	@setsid $(BIN_DIR)/$(BINARY_NAME) start --config broker.test.yaml > broker.log 2>&1 < /dev/null & echo $$! > broker.pid
+	@setsid $(BIN_DIR)/$(BINARY_NAME) start --config $(TEST_CONFIG) > broker.log 2>&1 < /dev/null & echo $$! > broker.pid
 	@sleep 5 # Give the broker a moment to start
 
 broker-stop: ## Stop the broker
@@ -97,8 +128,8 @@ broker-stop: ## Stop the broker
 	fi
 
 test-integration-go: ## Run Go integration tests
-	@echo "Running Go integration tests..."
+	@echo "Running Go integration tests (host=$(TEST_DB_HOST) port=$(TEST_DB_PORT))..."
-	@$(GO) test -v ./tests/integration/...
+	@TEST_DB_HOST=$(TEST_DB_HOST) TEST_DB_PORT=$(TEST_DB_PORT) $(GO) test -v ./tests/integration/...
 
 install: build ## Install the binary to GOPATH/bin
 	@echo "Installing to GOPATH/bin..."
@@ -172,6 +203,15 @@ docker-down: ## Stop PostgreSQL test database
 	fi
 	@echo "PostgreSQL stopped"
 
+docker-build: ## Build the pgsql-broker runtime image
+	@if [ "$(CONTAINER_RUNTIME)" = "none" ]; then echo "Error: Neither Docker nor Podman is installed"; exit 1; fi
+	@$(CONTAINER_RUNTIME) build \
+		--build-arg VERSION=$(VERSION) \
+		--build-arg COMMIT=$(COMMIT) \
+		--build-arg BUILD_TIME=$(BUILD_TIME) \
+		-t pgsql-broker:$(VERSION) -t pgsql-broker:latest \
+		-f Dockerfile .
+
 release: ## Create and push a new release tag (auto-increments patch version)
 	@echo "Creating new release..."
 	@latest_tag=$$(git describe --tags --abbrev=0 2>/dev/null || echo ""); \

README.md

@@ -1,6 +1,14 @@
 # PostgreSQL Broker
 
-A robust, event-driven job processing system for PostgreSQL that uses LISTEN/NOTIFY for real-time job execution. It supports multiple queues, priority-based scheduling, and can be used both as a standalone service or as a Go library.
+A robust, event-driven job processing system for PostgreSQL that uses LISTEN/NOTIFY for real-time job execution. It supports multiple queues, priority-based scheduling, multi-tenant row-level security, and can be used both as a standalone service or as a Go library.
+
+## Status
+[![Integration Tests](https://git.warky.dev/wdevs/pgsql-broker/actions/workflows/integration.yml/badge.svg?branch=main)](https://git.warky.dev/wdevs/pgsql-broker/actions?workflow=integration.yml)
+
+[![Release](https://git.warky.dev/wdevs/pgsql-broker/actions/workflows/release.yml/badge.svg?branch=main)](https://git.warky.dev/wdevs/pgsql-broker/actions?workflow=release.yml)
+
+[![Build & Release Docker Image](https://git.warky.dev/wdevs/pgsql-broker/actions/workflows/docker-release.yml/badge.svg?branch=main)](https://git.warky.dev/wdevs/pgsql-broker/actions?workflow=docker-release.yml)
+
 
 ## Features
 
@@ -8,20 +16,23 @@ A robust, event-driven job processing system for PostgreSQL that uses LISTEN/NOT
 - **Event-Driven**: Uses PostgreSQL LISTEN/NOTIFY for instant job notifications
 - **Multiple Queues**: Support for concurrent job processing across multiple queues per database
 - **Priority Scheduling**: Jobs can be prioritized for execution order
-- **Job Dependencies**: Jobs can depend on other jobs being completed first
+- **Job Dependencies**: Jobs can depend on other jobs completing first (cycle-checked, gated in `broker_get`)
+- **Multi-Tenant Row-Level Security**: Jobs are isolated per `tenant_id` via FORCE RLS policies (`broker.broker_set_tenant`)
+- **Lease-Based Claiming**: Jobs are claimed with an expiring lease and recovered automatically if a worker dies mid-job
+- **Least-Privilege Roles**: Optional `broker_admin` / `broker_runtime` / `broker_enqueue` role separation
+- **Versioned Migrations**: Embedded, ordered SQL migrations with an applied-versions tracking table
 - **Adapter Pattern**: Clean interfaces for database and logging (easy to extend)
 - **Standalone or Library**: Use as a CLI tool or integrate into your Go application
 - **Configuration Management**: Viper-based config with support for YAML, JSON, and environment variables
 - **Graceful Shutdown**: Proper cleanup and job completion on shutdown
-- **Instance Tracking**: Monitor active broker instances through the database
 - **Single Instance Per Database**: Enforces one broker instance per database to prevent conflicts
-- **Embedded SQL Installer**: Database schema embedded in binary with built-in install command
+- **Docker Support**: Production `Dockerfile`/`docker-compose.yml` and a test-runner `Dockerfile.test`/`docker-compose.test.yml`
 
 ## Architecture
 
 The broker supports multi-database architecture where a single broker process can manage multiple database connections. Each database has its own instance with dedicated queues, but only ONE broker instance is allowed per database.
 
-```
+```text
 ┌─────────────────────────────────────────────────────────────────┐
 │                      Broker Process                              │
 ├─────────────────────────────────────────────────────────────────┤
@@ -46,17 +57,20 @@ The broker supports multi-database architecture where a single broker process ca
     ┌──────────────────────┐      ┌──────────────────────┐
     │  PostgreSQL (DB1)    │      │  PostgreSQL (DB2)    │
     │  - broker_jobs       │      │  - broker_jobs       │
+    │  - broker_job_dependency│   │  - broker_job_dependency│
     │  - broker_queueinstance│    │  - broker_queueinstance│
     │  - broker_schedule   │      │  - broker_schedule   │
     └──────────────────────┘      └──────────────────────┘
 ```
 
 **Key Points**:
+
 - One broker process can manage multiple databases
 - Each database has exactly ONE active broker instance
 - Each database instance has its own queues and workers
 - Validation prevents multiple broker processes from connecting to the same database
 - Different databases can have different queue counts
+- Jobs are isolated by `tenant_id` via FORCE RLS; `broker_get` only claims jobs with no incomplete dependency
 
 ## Installation
 
@@ -76,27 +90,57 @@ The binary will be available in `bin/pgsql-broker`.
 go get git.warky.dev/wdevs/pgsql-broker
 ```
 
+### With Docker
+
+See [Docker](#docker) below for the production `Dockerfile`/`docker-compose.yml` and the `Dockerfile.test`/`docker-compose.test.yml` test runner.
+
 ## Quick Start
 
 ### 1. Setup Database
 
-Install the required tables and stored procedures:
+Migrations are embedded in the binary and applied in order, tracked in `broker.broker_schema_migrations`.
 
 ```bash
-# Using the CLI (recommended)
+# Apply all pending migrations, then verify
 ./bin/pgsql-broker install --config broker.yaml
 
+# Verify only, no changes
+./bin/pgsql-broker install --verify-only --config broker.yaml
+
 # Or with make
 make sql-install
-
-# Verify installation
-./bin/pgsql-broker install --verify-only --config broker.yaml
-
-# Or manually with psql:
-psql -f pkg/broker/install/sql/tables/00_install.sql
-psql -f pkg/broker/install/sql/procedures/00_install.sql
 ```
 
+#### Optional: least-privilege roles
+
+`install --with-roles` additionally creates/rotates three roles (idempotent — safe to re-run):
+
+| Role | Purpose |
+| ---- | ------- |
+| `broker_admin` | Schema owner, `BYPASSRLS`, runs migrations |
+| `broker_runtime` | The role the running broker connects as; table/sequence/function privileges only, no `BYPASSRLS` |
+| `broker_enqueue` | Narrow role for enqueuing jobs only (`broker_add_job`, `broker_set_tenant`, insert on `broker_jobs`/`broker_job_dependency`) |
+
+```bash
+./bin/pgsql-broker install --with-roles \
+  --admin-user postgres \
+  --broker-admin-password ... \
+  --broker-runtime-password ... \
+  --broker-enqueue-password ...
+```
+
+Credential resolution order for each value: CLI flag → environment variable → interactive masked prompt (TTY only) → error.
+
+| Value | Flag | Env fallback(s) |
+| ----- | ---- | ---------------- |
+| Admin user | `--admin-user` | `PGUSER`, `PG_USER` |
+| Admin password | `--admin-password` | `PGPASSWORD`, `PG_PASS` |
+| `broker_admin` password | `--broker-admin-password` | `BROKER_ADMIN_PASSWORD` |
+| `broker_runtime` password | `--broker-runtime-password` | `BROKER_RUNTIME_PASSWORD` |
+| `broker_enqueue` password | `--broker-enqueue-password` | `BROKER_ENQUEUE_PASSWORD` |
+
+`--with-roles` always connects as the admin login (not the config file's own user) since `broker_admin` must own the schema; it cannot be combined with `--verify-only`.
+
 ### 2. Configure
 
 Create a configuration file `broker.yaml`:
@@ -111,6 +155,7 @@ databases:
     password: your_password
     sslmode: disable
     queue_count: 4
+    tenant_id: default          # optional; sets broker.tenant_id for RLS
 
   # Optional: add more databases
   - name: db2
@@ -125,13 +170,15 @@ databases:
 broker:
   name: pgsql-broker
   enable_debug: false
+  lease_seconds: 60            # job lease duration before it's reclaimable
+  stale_job_recovery_sec: 30   # how often expired leases are recovered
 
 logging:
   level: info
   format: json
 ```
 
-**Note**: Each database requires a unique `name` identifier and can have its own `queue_count` configuration.
+**Note**: Each database requires a unique `name` identifier and can have its own `queue_count` configuration. See [broker.example.yaml](./broker.example.yaml) for the full field set.
 
 ### 3. Run the Broker
 
@@ -149,18 +196,47 @@ make run
 ### 4. Add a Job
 
 ```sql
-SELECT broker_add_job(
+SELECT * FROM broker.broker_add_job(
-    'My Job',                    -- job_name
+    'My Job',                    -- p_job_name
-    'SELECT do_something()',     -- execute_str
+    'SELECT do_something()',     -- p_execute_str
-    1,                           -- job_queue (default: 1)
+    1,                           -- p_job_queue (default: 1)
-    0,                           -- job_priority (default: 0)
+    0,                           -- p_job_priority (default: 0)
-    'sql',                       -- job_language (default: 'sql')
+    'sql',                       -- p_job_language (default: 'sql')
-    NULL,                        -- run_as
+    NULL,                        -- p_run_as
-    NULL,                        -- user_login
+    NULL,                        -- p_schedule_id
-    NULL                         -- schedule_id
+    ARRAY[123]::BIGINT[],        -- p_depends_on_job_ids (NULL = no deps)
+    NULL,                        -- p_idempotency_key
+    1,                           -- p_max_attempts (default: 1)
+    NULL,                        -- p_job_group (default: p_job_name)
+    ARRAY['extract']             -- p_depends_on_groups (NULL = no deps)
 );
 ```
 
+Or, for the common case (name, execute string, priority, dependencies), use the shortcut:
+
+```sql
+SELECT * FROM broker.broker_add_job_simple('transform', 'SELECT do_something()', 5, ARRAY['extract']);
+```
+
+#### Dependencies: job groups vs. job ids
+
+Every job belongs to a **job group**, defaulting to its own `job_name` when `p_job_group` isn't given. A dependency can target either a whole group (`p_depends_on_groups`) or a single job id (`p_depends_on_job_ids`):
+
+- **Group dependency** (fan-in): the dependent job is only claimable once **every** job currently tagged with that group has `complete_status = 2`. Multiple jobs can share a group (e.g. several parallel `extract` jobs feeding one `transform`); already-completed group members simply drop out of the check.
+- **Id dependency**: waits on one specific job row, as before.
+
+Both `broker_add_job` and `broker_add_job_simple` reject a job depending on its own group/id, and a one-hop cycle (A depends on B, B already depends on A).
+
+When calling as `broker_enqueue` (or any RLS-scoped role), set the tenant first and wrap in a single transaction so `SET LOCAL` persists across statements:
+
+```sql
+BEGIN;
+SET ROLE broker_enqueue;
+SELECT broker.broker_set_tenant('my_tenant');
+SELECT * FROM broker.broker_add_job_simple('extract', 'SELECT 1');
+COMMIT;
+```
+
 ## Usage as a Library
 
 ```go
@@ -194,34 +270,92 @@ func main() {
 
 See the [examples](./examples/) directory for complete examples.
 
+## Docker
+
+### Prebuilt image
+
+Published to `git.warky.dev/wdevs/pgsql-broker` on every `v*.*.*` tag (`.github/workflows/docker-release.yml`), tagged with the version and `latest`.
+
+```bash
+docker pull git.warky.dev/wdevs/pgsql-broker:latest
+docker run --rm -v $(pwd)/broker.yaml:/etc/pgsql-broker/broker.yaml:ro git.warky.dev/wdevs/pgsql-broker:latest
+```
+
+Config must be mounted at `/etc/pgsql-broker/broker.yaml` — no config is baked into the image.
+
+### Prebuilt image: `docker-compose.prebuilt.yml`
+
+Same stack as `docker-compose.yml` below, but pulls `git.warky.dev/wdevs/pgsql-broker:latest` instead of building from source — no local Go toolchain or Dockerfile needed.
+
+```bash
+cp broker.docker.example.yaml broker.docker.yaml   # set broker_runtime password
+cp .env.example .env                                # set POSTGRES_PASSWORD + 3 BROKER_*_PASSWORD
+docker-compose -f docker-compose.prebuilt.yml up -d
+```
+
+### Building the image locally
+
+```bash
+make docker-build   # builds pgsql-broker:<VERSION> and pgsql-broker:latest via Dockerfile
+```
+
+### Production: `Dockerfile` + `docker-compose.yml`
+
+Multi-stage build (`golang:1.26-alpine` → `alpine:3.22`, static binary, non-root user). The compose stack runs Postgres, a one-shot `migrate` service (`install --with-roles`), then the `broker` service connecting as `broker_runtime`.
+
+```bash
+cp broker.docker.example.yaml broker.docker.yaml   # set broker_runtime password
+cp .env.example .env                                # set POSTGRES_PASSWORD + 3 BROKER_*_PASSWORD
+docker-compose up --build -d
+```
+
+`broker.docker.yaml` and `.env` are gitignored — never commit real credentials. `BROKER_RUNTIME_PASSWORD` in `.env` must match the `password:` field in `broker.docker.yaml`.
+
+### Test runner: `Dockerfile.test` + `docker-compose.test.yml`
+
+Builds the module and runs the full Go integration suite (`tests/integration/...`) against a disposable Postgres container:
+
+```bash
+docker-compose -f docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from tests
+docker-compose -f docker-compose.test.yml down -v
+```
+
 ## Database Schema
 
+Schema objects live under `broker.*` and are applied via ordered, embedded migrations (`pkg/broker/install/sql/migrations/`), tracked in `broker.broker_schema_migrations`.
+
 ### Tables
 
 - **broker_queueinstance**: Tracks active broker queue instances (one per database)
-- **broker_jobs**: Job queue with status tracking
+- **broker_jobs**: Job queue with status, lease, attempt, tenant, and `job_group` tracking
+- **broker_job_dependency**: Dependency edges, each targeting either a `depends_on_job_id` or a `depends_on_group`, enforced at claim time
 - **broker_schedule**: Scheduled jobs (cron-like functionality)
 
 ### Stored Procedures
 
-- **broker_get**: Fetch the next job from a queue
+- **broker_get**: Fetch and lease the next claimable job from a queue (skips jobs with an incomplete id- or group-dependency)
 - **broker_run**: Execute a job
 - **broker_set**: Set runtime options (user, application_name, etc.)
-- **broker_add_job**: Add a new job to the queue
+- **broker_set_tenant**: Set the `broker.tenant_id` GUC used by RLS policies
-- **broker_register_instance**: Register a broker instance
+- **broker_add_job**: Add a new job to the queue, with an optional `job_group` and dependencies by id and/or group
-- **broker_ping_instance**: Update instance heartbeat
+- **broker_add_job_simple**: Shortcut for `broker_add_job` — name, execute string, priority, and dependencies by group name
-- **broker_shutdown_instance**: Mark instance as shutdown
+- **broker_register_instance / broker_ping_instance / broker_shutdown_instance**: Instance lifecycle tracking
+- **broker_recover_stale_jobs**: Reclaim jobs whose lease has expired
+
+### Roles (optional, via `install --with-roles`)
+
+See `pkg/broker/install/sql/roles/0001_roles.sql` — `broker_admin`, `broker_runtime`, `broker_enqueue` (see table above).
 
 ## Configuration Reference
 
-See [broker.example.yaml](./broker.example.yaml) for a complete configuration example.
+See [broker.example.yaml](./broker.example.yaml) for a complete configuration example, or [broker.docker.example.yaml](./broker.docker.example.yaml) for the Docker Compose variant.
 
 ### Database Settings
 
 The `databases` array can contain multiple database configurations. Each entry supports:
 
 | Setting | Description | Default |
-|---------|-------------|---------|
+| ------- | ----------- | ------- |
 | `name` | Unique identifier for this database | **Required** |
 | `host` | PostgreSQL host | `localhost` |
 | `port` | PostgreSQL port | `5432` |
@@ -234,13 +368,35 @@ The `databases` array can contain multiple database configurations. Each entry s
 | `conn_max_lifetime` | Connection max lifetime | `5m` |
 | `conn_max_idle_time` | Connection max idle time | `10m` |
 | `queue_count` | Number of queues for this database | `4` |
+| `tenant_id` | Tenant id set for this connection (RLS) | `default` |
+| `auto_migrate` | Apply pending migrations during `start` | `false` |
+| `disabled` | Exclude this database from `start` | `false` |
+
+### Database Management Commands
+
+The configuration can be managed without editing YAML by hand:
+
+```bash
+pgsql-broker db list --config broker.yaml
+pgsql-broker db add replica --from db1 --host db.example --database jobs_replica --user broker --password secret --yes --non-interactive --config broker.yaml
+pgsql-broker db disable replica --config broker.yaml
+pgsql-broker db enable replica --config broker.yaml
+pgsql-broker db remove replica --yes --config broker.yaml
+```
+
+`db add` accepts flags for every database setting. If a setting is omitted in
+interactive mode, it is pre-filled from the last database in the file, or
+from the instance named by `--from`; press Enter to keep that value. Use
+`--non-interactive` to reject missing required values instead of prompting.
+The add command asks for confirmation unless `--yes` is supplied. Remove also
+requires `--yes` when stdin is not a terminal.
 
 ### Broker Settings
 
 Global settings applied to all database instances:
 
 | Setting | Description | Default |
-|---------|-------------|---------|
+| ------- | ----------- | ------- |
 | `name` | Broker instance name | `pgsql-broker` |
 | `fetch_query_que_size` | Jobs per fetch cycle | `100` |
 | `queue_timer_sec` | Seconds between polls | `10` |
@@ -248,6 +404,12 @@ Global settings applied to all database instances:
 | `worker_idle_timeout_sec` | Worker idle timeout | `10` |
 | `notify_retry_seconds` | NOTIFY retry interval | `30s` |
 | `enable_debug` | Enable debug logging | `false` |
+| `lease_seconds` | Job lease duration before reclaimable | `60` |
+| `stale_job_recovery_sec` | Interval for reclaiming expired leases | `30` |
+| `metrics_enabled` | Enable Prometheus and the embedded dashboard | `true` |
+| `metrics_host` | Metrics HTTP bind host | `0.0.0.0` |
+| `metrics_port` | Metrics HTTP bind port | `9469` |
+| `queue_depth_poll_sec` | Queue depth collection interval | `15` |
 
 ## Development
 
@@ -262,9 +424,19 @@ make vet         # Run go vet
 make test        # Run tests
 ```
 
+### Testing
+
+```bash
+go test ./...                          # unit tests
+go test -v ./tests/integration/...     # integration tests (needs local Postgres, see below)
+docker-compose -f docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from tests
+```
+
+Integration tests expect Postgres reachable at `127.0.0.1:5433` (see `tests/integration/`), including `rls_test.go` (multi-tenant isolation) and `stage5_test.go`. `127.0.0.1` is used instead of `localhost` because some CI Docker hosts publish container ports on IPv4 only, and `localhost` can resolve to `::1` first and fail.
+
 ### Project Structure
 
-```
+```text
 pgsql-broker/
 ├── cmd/broker/          # CLI application
 ├── pkg/broker/          # Core broker package
@@ -274,16 +446,22 @@ pgsql-broker/
 │   ├── queue/           # Queue management
 │   ├── worker/          # Worker implementation
 │   └── install/         # Database installer with embedded SQL
-│       └── sql/         # SQL schema (embedded in binary)
+│       └── sql/
-│           ├── tables/  # Table definitions
+│           ├── migrations/ # Ordered, versioned schema migrations (embedded in binary)
-│           └── procedures/ # Stored procedures
+│           └── roles/      # Optional least-privilege role DDL (--with-roles)
+├── tests/integration/   # Go integration test suite
 ├── examples/            # Usage examples
+├── Dockerfile           # Production broker image
+├── Dockerfile.test      # Integration test runner image
+├── docker-compose.yml   # Postgres + migrate + broker
+├── docker-compose.test.yml # Postgres + test runner
 └── Makefile             # Build automation
 ```
 
 ## Contributing
 
 Contributions are welcome! Please ensure:
+
 - Code is formatted with `go fmt`
 - Tests pass with `go test ./...`
 - Documentation is updated

broker.docker.example.yaml

@@ -0,0 +1,33 @@
+# Config for the docker-compose.yml stack. Copy to broker.docker.yaml
+# (already gitignored), fill in a real password, and it is mounted
+# read-only into the broker/migrate containers at
+# /etc/pgsql-broker/broker.yaml.
+
+databases:
+  - name: primary
+    host: postgres
+    port: 5432
+    database: broker
+    user: broker_runtime
+    password: change_me
+    sslmode: disable
+    max_open_conns: 25
+    max_idle_conns: 5
+    conn_max_lifetime: 5m
+    conn_max_idle_time: 10m
+    queue_count: 4
+
+broker:
+  name: pgsql-broker
+  fetch_query_que_size: 100
+  queue_timer_sec: 10
+  queue_buffer_size: 50
+  worker_idle_timeout_sec: 10
+  notify_retry_seconds: 30s
+  enable_debug: false
+  lease_seconds: 60
+  stale_job_recovery_sec: 30
+
+logging:
+  level: info
+  format: json

broker.example.yaml

@@ -41,6 +41,10 @@ broker:
   worker_idle_timeout_sec: 10  # Worker idle timeout
   notify_retry_seconds: 30s  # LISTEN/NOTIFY retry interval
   enable_debug: false  # Enable debug logging
+  metrics_enabled: true # Expose Prometheus and the embedded dashboard
+  metrics_host: 0.0.0.0
+  metrics_port: 9469
+  queue_depth_poll_sec: 15
 
 # Logging settings
 logging:

broker.test.yaml

@@ -1,6 +1,6 @@
 databases:
   - name: test
-    host: localhost
+    host: 127.0.0.1
     port: 5433
     database: broker_test
     user: user

cmd/broker/db.go

@@ -0,0 +1,513 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/term"
+
+	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/config"
+)
+
+// Flags shared by the `db` subcommands.
+var (
+	dbConfigPath string
+
+	dbAddFrom            string
+	dbAddName            string
+	dbAddHost            string
+	dbAddPort            int
+	dbAddDatabase        string
+	dbAddUser            string
+	dbAddPassword        string
+	dbAddSSLMode         string
+	dbAddMaxOpenConns    int
+	dbAddMaxIdleConns    int
+	dbAddConnMaxLifetime string
+	dbAddConnMaxIdleTime string
+	dbAddQueueCount      int
+	dbAddTenantID        string
+	dbAddAutoMigrate     bool
+	dbAddDisabled        bool
+	dbAddNonInteractive  bool
+	dbYes                bool
+)
+
+var dbCmd = &cobra.Command{
+	Use:   "db",
+	Short: "Manage databases in the broker config file",
+	Long:  `Add, remove, enable, disable, or list the databases configured in the broker config file.`,
+}
+
+var dbAddCmd = &cobra.Command{
+	Use:   "add [name]",
+	Short: "Add a database to the config file",
+	Long: `Add a database entry to the config file.
+
+Every field can be supplied via flags for fully non-interactive use. Any
+field left unset falls back to an interactive prompt (when stdin is a
+terminal) that is pre-filled with the value from --from (a named existing
+instance) or, if --from is not given, the last database currently in the
+file. Press Enter at a prompt to accept the shown value.`,
+	Args: cobra.MaximumNArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if len(args) == 1 && dbAddName == "" {
+			dbAddName = args[0]
+		}
+		return runDBAdd(cmd)
+	},
+}
+
+var dbRemoveCmd = &cobra.Command{
+	Use:   "remove <name>",
+	Short: "Remove a database from the config file",
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runDBRemove(args[0])
+	},
+}
+
+var dbEnableCmd = &cobra.Command{
+	Use:   "enable <name>",
+	Short: "Enable a database (removes the disabled flag)",
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runDBSetDisabled(args[0], false)
+	},
+}
+
+var dbDisableCmd = &cobra.Command{
+	Use:   "disable <name>",
+	Short: "Disable a database (excluded from `start` until re-enabled)",
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runDBSetDisabled(args[0], true)
+	},
+}
+
+var dbListCmd = &cobra.Command{
+	Use:   "list",
+	Short: "List databases configured in the config file",
+	Args:  cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runDBList()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(dbCmd)
+	dbCmd.AddCommand(dbAddCmd, dbRemoveCmd, dbEnableCmd, dbDisableCmd, dbListCmd)
+
+	dbCmd.PersistentFlags().StringVar(&dbConfigPath, "config", "broker.yaml", "config file to edit")
+
+	dbAddCmd.Flags().StringVar(&dbAddFrom, "from", "", "prime defaults from this existing database name (default: last database in the file)")
+	dbAddCmd.Flags().StringVar(&dbAddName, "name", "", "unique name for the new database (required)")
+	dbAddCmd.Flags().StringVar(&dbAddHost, "host", "", "PostgreSQL host")
+	dbAddCmd.Flags().IntVar(&dbAddPort, "port", 0, "PostgreSQL port (default 5432)")
+	dbAddCmd.Flags().StringVar(&dbAddDatabase, "database", "", "database name")
+	dbAddCmd.Flags().StringVar(&dbAddUser, "user", "", "database user")
+	dbAddCmd.Flags().StringVar(&dbAddPassword, "password", "", "database password")
+	dbAddCmd.Flags().StringVar(&dbAddSSLMode, "sslmode", "", "SSL mode (disable, require, verify-ca, verify-full)")
+	dbAddCmd.Flags().IntVar(&dbAddMaxOpenConns, "max-open-conns", 0, "max open connections (default 25)")
+	dbAddCmd.Flags().IntVar(&dbAddMaxIdleConns, "max-idle-conns", 0, "max idle connections (default 5)")
+	dbAddCmd.Flags().StringVar(&dbAddConnMaxLifetime, "conn-max-lifetime", "", "connection max lifetime (e.g. 5m, default 5m)")
+	dbAddCmd.Flags().StringVar(&dbAddConnMaxIdleTime, "conn-max-idle-time", "", "connection max idle time (e.g. 10m, default 10m)")
+	dbAddCmd.Flags().IntVar(&dbAddQueueCount, "queue-count", 0, "number of concurrent queues (default 4)")
+	dbAddCmd.Flags().StringVar(&dbAddTenantID, "tenant-id", "", "RLS tenant id (default \"default\")")
+	dbAddCmd.Flags().BoolVar(&dbAddAutoMigrate, "auto-migrate", false, "apply pending migrations automatically on connect")
+	dbAddCmd.Flags().BoolVar(&dbAddDisabled, "disabled", false, "add the database in a disabled state")
+	dbAddCmd.Flags().BoolVar(&dbAddNonInteractive, "non-interactive", false, "fail instead of prompting for missing fields")
+	dbAddCmd.Flags().BoolVarP(&dbYes, "yes", "y", false, "skip the confirmation prompt")
+
+	dbRemoveCmd.Flags().BoolVarP(&dbYes, "yes", "y", false, "skip the confirmation prompt")
+}
+
+// runDBAdd builds a new DatabaseConfig by layering (in priority order)
+// explicit flags, then interactive prompts primed from --from / the last
+// database in the file, then falls back to leaving optional fields unset
+// so config.applyDatabaseDefaults keeps handling defaults at load time.
+func runDBAdd(cmd *cobra.Command) error {
+	doc, err := config.LoadFileDoc(dbConfigPath)
+	if err != nil {
+		return err
+	}
+
+	var prime config.DatabaseConfig
+	if dbAddFrom != "" {
+		found, ok, err := doc.FindDatabase(dbAddFrom)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			return fmt.Errorf("--from instance %q not found in %s", dbAddFrom, dbConfigPath)
+		}
+		prime = found
+	} else if last, ok, err := doc.LastDatabase(); err != nil {
+		return err
+	} else if ok {
+		prime = last
+	}
+	// Never prime the new entry's name from an existing one.
+	prime.Name = ""
+
+	interactive := !dbAddNonInteractive && term.IsTerminal(int(os.Stdin.Fd()))
+	reader := bufio.NewReader(os.Stdin)
+
+	if interactive {
+		fmt.Fprintln(os.Stderr, "Adding a new database. Press Enter to accept the shown value.")
+	}
+
+	db := config.DatabaseConfig{}
+
+	if dbAddName != "" {
+		db.Name = dbAddName
+	} else {
+		db.Name, err = resolveStringField(cmd, reader, interactive, "name", "", true, dbAddName)
+		if err != nil {
+			return err
+		}
+	}
+	if db.Name == "" {
+		return fmt.Errorf("name is required (--name, a positional argument, or run interactively)")
+	}
+	db.Host, err = resolveStringField(cmd, reader, interactive, "host", prime.Host, true, dbAddHost)
+	if err != nil {
+		return err
+	}
+	db.Database, err = resolveStringField(cmd, reader, interactive, "database", prime.Database, true, dbAddDatabase)
+	if err != nil {
+		return err
+	}
+	db.User, err = resolveStringField(cmd, reader, interactive, "user", prime.User, true, dbAddUser)
+	if err != nil {
+		return err
+	}
+	db.Password, err = resolveStringField(cmd, reader, interactive, "password", prime.Password, false, dbAddPassword)
+	if err != nil {
+		return err
+	}
+	db.SSLMode, err = resolveStringField(cmd, reader, interactive, "sslmode", prime.SSLMode, false, dbAddSSLMode)
+	if err != nil {
+		return err
+	}
+	db.TenantID, err = resolveStringField(cmd, reader, interactive, "tenant_id", prime.TenantID, false, dbAddTenantID)
+	if err != nil {
+		return err
+	}
+
+	port, err := resolveIntField(cmd, reader, interactive, "port", prime.Port, "port", dbAddPort)
+	if err != nil {
+		return err
+	}
+	db.Port = port
+
+	maxOpen, err := resolveIntField(cmd, reader, interactive, "max_open_conns", prime.MaxOpenConns, "max-open-conns", dbAddMaxOpenConns)
+	if err != nil {
+		return err
+	}
+	db.MaxOpenConns = maxOpen
+
+	maxIdle, err := resolveIntField(cmd, reader, interactive, "max_idle_conns", prime.MaxIdleConns, "max-idle-conns", dbAddMaxIdleConns)
+	if err != nil {
+		return err
+	}
+	db.MaxIdleConns = maxIdle
+
+	queueCount, err := resolveIntField(cmd, reader, interactive, "queue_count", prime.QueueCount, "queue-count", dbAddQueueCount)
+	if err != nil {
+		return err
+	}
+	db.QueueCount = queueCount
+
+	connMaxLifetime, err := resolveDurationField(cmd, reader, interactive, "conn_max_lifetime", prime.ConnMaxLifetime, "conn-max-lifetime", dbAddConnMaxLifetime)
+	if err != nil {
+		return err
+	}
+	db.ConnMaxLifetime = connMaxLifetime
+
+	connMaxIdleTime, err := resolveDurationField(cmd, reader, interactive, "conn_max_idle_time", prime.ConnMaxIdleTime, "conn-max-idle-time", dbAddConnMaxIdleTime)
+	if err != nil {
+		return err
+	}
+	db.ConnMaxIdleTime = connMaxIdleTime
+
+	if cmd.Flags().Changed("auto-migrate") {
+		db.AutoMigrate = dbAddAutoMigrate
+	} else {
+		db.AutoMigrate = prime.AutoMigrate
+	}
+	if cmd.Flags().Changed("disabled") {
+		db.Disabled = dbAddDisabled
+	} else {
+		db.Disabled = prime.Disabled
+	}
+
+	for field, value := range map[string]string{
+		"host":     db.Host,
+		"database": db.Database,
+		"user":     db.User,
+	} {
+		if strings.TrimSpace(value) == "" {
+			return fmt.Errorf("%s is required (--%s or an interactive value)", field, strings.ReplaceAll(field, "_", "-"))
+		}
+	}
+
+	if err := doc.AddDatabase(db); err != nil {
+		return err
+	}
+
+	if !dbYes && interactive {
+		fmt.Fprintf(os.Stderr, "\nAbout to add database %q to %s:\n", db.Name, dbConfigPath)
+		printDBSummary(os.Stderr, db)
+		fmt.Fprint(os.Stderr, "Proceed? [Y/n]: ")
+		line, _ := reader.ReadString('\n')
+		line = strings.TrimSpace(strings.ToLower(line))
+		if line == "n" || line == "no" {
+			fmt.Println("Aborted; config file not modified.")
+			return nil
+		}
+	}
+
+	if err := doc.Save(); err != nil {
+		return err
+	}
+	fmt.Printf("Added database %q to %s\n", db.Name, dbConfigPath)
+	return nil
+}
+
+func runDBRemove(name string) error {
+	doc, err := config.LoadFileDoc(dbConfigPath)
+	if err != nil {
+		return err
+	}
+	db, ok, err := doc.FindDatabase(name)
+	if err != nil {
+		return err
+	}
+	if !ok {
+		return fmt.Errorf("database %q not found in %s", name, dbConfigPath)
+	}
+
+	if !dbYes {
+		if !term.IsTerminal(int(os.Stdin.Fd())) {
+			return fmt.Errorf("refusing to remove %q non-interactively without --yes", name)
+		}
+		fmt.Fprintf(os.Stderr, "Remove database %q (%s@%s:%d/%s) from %s? [y/N]: ",
+			name, db.User, db.Host, db.Port, db.Database, dbConfigPath)
+		reader := bufio.NewReader(os.Stdin)
+		line, _ := reader.ReadString('\n')
+		line = strings.TrimSpace(strings.ToLower(line))
+		if line != "y" && line != "yes" {
+			fmt.Println("Aborted; config file not modified.")
+			return nil
+		}
+	}
+
+	if _, err := doc.RemoveDatabase(name); err != nil {
+		return err
+	}
+	if err := doc.Save(); err != nil {
+		return err
+	}
+	fmt.Printf("Removed database %q from %s\n", name, dbConfigPath)
+	return nil
+}
+
+func runDBSetDisabled(name string, disabled bool) error {
+	doc, err := config.LoadFileDoc(dbConfigPath)
+	if err != nil {
+		return err
+	}
+	found, err := doc.SetDisabled(name, disabled)
+	if err != nil {
+		return err
+	}
+	if !found {
+		return fmt.Errorf("database %q not found in %s", name, dbConfigPath)
+	}
+	if err := doc.Save(); err != nil {
+		return err
+	}
+	verb := "Enabled"
+	if disabled {
+		verb = "Disabled"
+	}
+	fmt.Printf("%s database %q in %s\n", verb, name, dbConfigPath)
+	return nil
+}
+
+func runDBList() error {
+	doc, err := config.LoadFileDoc(dbConfigPath)
+	if err != nil {
+		return err
+	}
+	dbs, err := doc.ListDatabases()
+	if err != nil {
+		return err
+	}
+	if len(dbs) == 0 {
+		fmt.Printf("No databases configured in %s\n", dbConfigPath)
+		return nil
+	}
+	fmt.Printf("%-20s %-20s %-6s %-16s %-16s %-10s\n", "NAME", "HOST", "PORT", "DATABASE", "USER", "STATUS")
+	for i := range dbs {
+		db := &dbs[i]
+		status := "enabled"
+		if db.Disabled {
+			status = "disabled"
+		}
+		port := db.Port
+		if port == 0 {
+			port = 5432
+		}
+		fmt.Printf("%-20s %-20s %-6d %-16s %-16s %-10s\n", db.Name, db.Host, port, db.Database, db.User, status)
+	}
+	return nil
+}
+
+func printDBSummary(w *os.File, db config.DatabaseConfig) {
+	password := ""
+	if db.Password != "" {
+		password = "(set)"
+	}
+	fmt.Fprintf(w, "  host:              %s\n", db.Host)
+	fmt.Fprintf(w, "  port:              %s\n", intOrDefault(db.Port, 5432))
+	fmt.Fprintf(w, "  database:          %s\n", db.Database)
+	fmt.Fprintf(w, "  user:              %s\n", db.User)
+	fmt.Fprintf(w, "  password:          %s\n", password)
+	fmt.Fprintf(w, "  sslmode:           %s\n", stringOrDefault(db.SSLMode, "disable"))
+	fmt.Fprintf(w, "  max_open_conns:    %s\n", intOrDefault(db.MaxOpenConns, 25))
+	fmt.Fprintf(w, "  max_idle_conns:    %s\n", intOrDefault(db.MaxIdleConns, 5))
+	fmt.Fprintf(w, "  conn_max_lifetime: %s\n", durationOrDefault(db.ConnMaxLifetime, 5*time.Minute))
+	fmt.Fprintf(w, "  conn_max_idle_time:%s\n", durationOrDefault(db.ConnMaxIdleTime, 10*time.Minute))
+	fmt.Fprintf(w, "  queue_count:       %s\n", intOrDefault(db.QueueCount, 4))
+	fmt.Fprintf(w, "  tenant_id:         %s\n", stringOrDefault(db.TenantID, "default"))
+	fmt.Fprintf(w, "  auto_migrate:      %t\n", db.AutoMigrate)
+	fmt.Fprintf(w, "  disabled:          %t\n", db.Disabled)
+}
+
+func intOrDefault(v, def int) string {
+	if v == 0 {
+		return fmt.Sprintf("%d (default)", def)
+	}
+	return strconv.Itoa(v)
+}
+
+func stringOrDefault(v, def string) string {
+	if v == "" {
+		return fmt.Sprintf("%s (default)", def)
+	}
+	return v
+}
+
+func durationOrDefault(v, def time.Duration) string {
+	if v == 0 {
+		return fmt.Sprintf("%s (default)", def)
+	}
+	return v.String()
+}
+
+// resolveStringField returns, in priority order: the flag value if the
+// flag was explicitly set, otherwise an interactive prompt (pre-filled
+// with defaultVal) when interactive, otherwise defaultVal. When required
+// and still empty after all of that, an error is returned by the caller.
+func resolveStringField(cmd *cobra.Command, reader *bufio.Reader, interactive bool, label, defaultVal string, required bool, flagVal string) (string, error) {
+	flagName := flagNameFor(label)
+	if cmd.Flags().Changed(flagName) {
+		return flagVal, nil
+	}
+	if !interactive {
+		return defaultVal, nil
+	}
+	for {
+		fmt.Fprintf(os.Stderr, "%s [%s]: ", label, defaultVal)
+		line, _ := reader.ReadString('\n')
+		line = strings.TrimSpace(line)
+		if line == "" {
+			line = defaultVal
+		}
+		if line == "" && required {
+			fmt.Fprintln(os.Stderr, "value is required")
+			continue
+		}
+		return line, nil
+	}
+}
+
+func resolveIntField(cmd *cobra.Command, reader *bufio.Reader, interactive bool, label string, defaultVal int, flagName string, flagVal int) (int, error) {
+	if cmd.Flags().Changed(flagName) {
+		return flagVal, nil
+	}
+	if !interactive {
+		return defaultVal, nil
+	}
+	for {
+		fmt.Fprintf(os.Stderr, "%s [%d]: ", label, defaultVal)
+		line, _ := reader.ReadString('\n')
+		line = strings.TrimSpace(line)
+		if line == "" {
+			return defaultVal, nil
+		}
+		n, err := strconv.Atoi(line)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "must be a whole number")
+			continue
+		}
+		return n, nil
+	}
+}
+
+func resolveDurationField(cmd *cobra.Command, reader *bufio.Reader, interactive bool, label string, defaultVal time.Duration, flagName, flagVal string) (time.Duration, error) {
+	if cmd.Flags().Changed(flagName) {
+		d, err := time.ParseDuration(flagVal)
+		if err != nil {
+			return 0, fmt.Errorf("--%s: %w", flagName, err)
+		}
+		return d, nil
+	}
+	if !interactive {
+		return defaultVal, nil
+	}
+	for {
+		fmt.Fprintf(os.Stderr, "%s [%s]: ", label, defaultVal)
+		line, _ := reader.ReadString('\n')
+		line = strings.TrimSpace(line)
+		if line == "" {
+			return defaultVal, nil
+		}
+		d, err := time.ParseDuration(line)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "must be a duration like 5m, 30s")
+			continue
+		}
+		return d, nil
+	}
+}
+
+// flagNameFor maps a prompt label to its cobra flag name for the string
+// fields prompted in runDBAdd.
+func flagNameFor(label string) string {
+	switch label {
+	case "name":
+		return "name"
+	case "host":
+		return "host"
+	case "database":
+		return "database"
+	case "user":
+		return "user"
+	case "password":
+		return "password"
+	case "sslmode":
+		return "sslmode"
+	case "tenant_id":
+		return "tenant-id"
+	default:
+		return label
+	}
+}

cmd/broker/main.go

@@ -6,9 +6,12 @@ import (
 	"log/slog"
 	"os"
 	"os/signal"
+	"runtime/debug"
 	"syscall"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/term"
+
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker"
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/config"
@@ -25,6 +28,13 @@ var (
 	cfgFile    string
 	logLevel   string
 	verifyOnly bool
+
+	withRoles             bool
+	adminUser             string
+	adminPassword         string
+	brokerAdminPassword   string
+	brokerRuntimePassword string
+	brokerEnqueuePassword string
 )
 
 func main() {
@@ -83,9 +93,59 @@ func init() {
 
 	// Install command flags
 	installCmd.Flags().BoolVar(&verifyOnly, "verify-only", false, "only verify installation without installing")
+	installCmd.Flags().BoolVar(&withRoles, "with-roles", false, "also create/update the broker_admin, broker_runtime, and broker_enqueue roles")
+	installCmd.Flags().StringVar(&adminUser, "admin-user", "", "superuser/CREATEROLE login used only for --with-roles (falls back to PGUSER/PG_USER env)")
+	installCmd.Flags().StringVar(&adminPassword, "admin-password", "", "password for --admin-user (falls back to PGPASSWORD/PG_PASS env, then an interactive prompt)")
+	installCmd.Flags().StringVar(&brokerAdminPassword, "broker-admin-password", "", "password to set for broker_admin (falls back to BROKER_ADMIN_PASSWORD env, then an interactive prompt)")
+	installCmd.Flags().StringVar(&brokerRuntimePassword, "broker-runtime-password", "", "password to set for broker_runtime (falls back to BROKER_RUNTIME_PASSWORD env, then an interactive prompt)")
+	installCmd.Flags().StringVar(&brokerEnqueuePassword, "broker-enqueue-password", "", "password to set for broker_enqueue (falls back to BROKER_ENQUEUE_PASSWORD env, then an interactive prompt)")
 }
 
-func runBroker() error {
+// resolveCredential returns the first non-empty value among the flag value,
+// the given environment variables (checked in order), and -- if none are
+// set -- an interactive masked prompt. It errors rather than prompting when
+// stdin is not a terminal, since a hang in a non-interactive context (CI,
+// systemd) is worse than a clear failure.
+func resolveCredential(flagVal string, envNames []string, promptLabel string) (string, error) {
+	if flagVal != "" {
+		return flagVal, nil
+	}
+	for _, name := range envNames {
+		if v := os.Getenv(name); v != "" {
+			return v, nil
+		}
+	}
+
+	fd := int(os.Stdin.Fd())
+	if !term.IsTerminal(fd) {
+		return "", fmt.Errorf(
+			"%s not provided and stdin is not a terminal to prompt on; pass it via flag or one of %v",
+			promptLabel, envNames,
+		)
+	}
+
+	fmt.Fprintf(os.Stderr, "%s: ", promptLabel)
+	b, err := term.ReadPassword(fd)
+	fmt.Fprintln(os.Stderr)
+	if err != nil {
+		return "", fmt.Errorf("failed to read %s: %w", promptLabel, err)
+	}
+	if len(b) == 0 {
+		return "", fmt.Errorf("%s must not be empty", promptLabel)
+	}
+	return string(b), nil
+}
+
+func runBroker() (err error) {
+	// Top-level safety net: an unrecovered panic anywhere in startup or the
+	// shutdown wait must not crash the process with a raw trace -- log it
+	// and return a normal error instead.
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("recovered from panic in runBroker: %v\n%s", r, debug.Stack())
+		}
+	}()
+
 	// Load configuration
 	cfg, err := config.LoadConfig(cfgFile)
 	if err != nil {
@@ -136,12 +196,51 @@ func runInstall() error {
 
 	ctx := context.Background()
 
+	var rolePasswords install.RolePasswords
+	var adminUserVal, adminPasswordVal string
+	if withRoles {
+		if verifyOnly {
+			return fmt.Errorf("--with-roles cannot be combined with --verify-only")
+		}
+
+		var err error
+		adminUserVal, err = resolveCredential(adminUser, []string{"PGUSER", "PG_USER"}, "admin user (superuser/CREATEROLE login for --with-roles)")
+		if err != nil {
+			return err
+		}
+		adminPasswordVal, err = resolveCredential(adminPassword, []string{"PGPASSWORD", "PG_PASS"}, "admin password")
+		if err != nil {
+			return err
+		}
+		rolePasswords.AdminPassword, err = resolveCredential(brokerAdminPassword, []string{"BROKER_ADMIN_PASSWORD"}, "broker_admin password")
+		if err != nil {
+			return err
+		}
+		rolePasswords.RuntimePassword, err = resolveCredential(brokerRuntimePassword, []string{"BROKER_RUNTIME_PASSWORD"}, "broker_runtime password")
+		if err != nil {
+			return err
+		}
+		rolePasswords.EnqueuePassword, err = resolveCredential(brokerEnqueuePassword, []string{"BROKER_ENQUEUE_PASSWORD"}, "broker_enqueue password")
+		if err != nil {
+			return err
+		}
+	}
+
 	// Install/verify on all configured databases
-	for i, dbCfg := range cfg.Databases {
+	for i := range cfg.Databases {
+		dbCfg := &cfg.Databases[i]
 		logger.Info("processing database", "index", i, "name", dbCfg.Name, "host", dbCfg.Host, "database", dbCfg.Database)
 
-		// Create database adapter
+		// Create database adapter. With --with-roles, the config file's own
-		dbAdapter := adapter.NewPostgresAdapter(dbCfg.ToPostgresConfig(), logger)
+		// user (typically the least-privilege broker_runtime) may not exist
+		// yet on a fresh cluster -- migrations and role creation both run as
+		// the admin login instead, since broker_admin must own the schema.
+		pgCfg := dbCfg.ToPostgresConfig()
+		if withRoles {
+			pgCfg.User = adminUserVal
+			pgCfg.Password = adminPasswordVal
+		}
+		dbAdapter := adapter.NewPostgresAdapter(pgCfg, logger)
 
 		// Connect to database
 		if err := dbAdapter.Connect(ctx); err != nil {
@@ -161,9 +260,9 @@ func runInstall() error {
 			}
 			logger.Info("database schema verified successfully", "database", dbCfg.Name)
 		} else {
-			// Install schema
+			// Apply migrations
-			logger.Info("installing database schema", "database", dbCfg.Name)
+			logger.Info("applying database migrations", "database", dbCfg.Name)
-			if err := installer.InstallSchema(ctx); err != nil {
+			if err := installer.ApplyMigrations(ctx); err != nil {
 				dbAdapter.Close()
 				logger.Error("installation failed", "database", dbCfg.Name, "error", err)
 				return fmt.Errorf("installation failed for %s: %w", dbCfg.Name, err)
@@ -180,6 +279,15 @@ func runInstall() error {
 			logger.Info("database schema installed and verified successfully", "database", dbCfg.Name)
 		}
 
+		if withRoles && !verifyOnly {
+			logger.Info("applying roles", "database", dbCfg.Name)
+			if err := installer.InstallRoles(ctx, rolePasswords); err != nil {
+				dbAdapter.Close()
+				return fmt.Errorf("failed to install roles for %s: %w", dbCfg.Name, err)
+			}
+			logger.Info("roles installed successfully", "database", dbCfg.Name)
+		}
+
 		dbAdapter.Close()
 	}
 

docker-compose.prebuilt.yml

@@ -0,0 +1,46 @@
+services:
+  postgres:
+    image: docker.io/library/postgres:16-alpine
+    environment:
+      POSTGRES_DB: broker
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres -d broker"]
+      interval: 2s
+      timeout: 3s
+      retries: 30
+    restart: unless-stopped
+
+  # One-shot: applies migrations and creates/rotates the least-privilege
+  # broker_admin/broker_runtime/broker_enqueue roles, then exits. The
+  # broker service below only starts once this completes successfully.
+  migrate:
+    image: git.warky.dev/wdevs/pgsql-broker:latest
+    depends_on:
+      postgres:
+        condition: service_healthy
+    volumes:
+      - ./broker.docker.yaml:/etc/pgsql-broker/broker.yaml:ro
+    environment:
+      PGUSER: postgres
+      PGPASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
+      BROKER_ADMIN_PASSWORD: ${BROKER_ADMIN_PASSWORD:?set BROKER_ADMIN_PASSWORD in .env}
+      BROKER_RUNTIME_PASSWORD: ${BROKER_RUNTIME_PASSWORD:?set BROKER_RUNTIME_PASSWORD in .env}
+      BROKER_ENQUEUE_PASSWORD: ${BROKER_ENQUEUE_PASSWORD:?set BROKER_ENQUEUE_PASSWORD in .env}
+    command: ["install", "--with-roles"]
+    restart: "no"
+
+  broker:
+    image: git.warky.dev/wdevs/pgsql-broker:latest
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+    volumes:
+      - ./broker.docker.yaml:/etc/pgsql-broker/broker.yaml:ro
+    restart: unless-stopped
+
+volumes:
+  postgres-data:

docker-compose.test.yml

@@ -0,0 +1,26 @@
+services:
+  postgres:
+    image: docker.io/library/postgres:16-alpine
+    # Tests hardcode host=localhost port=5433, so postgres listens on 5433
+    # internally and the tests container joins its network namespace below.
+    command: ["postgres", "-p", "5433"]
+    environment:
+      POSTGRES_DB: broker_test
+      POSTGRES_USER: user
+      POSTGRES_PASSWORD: password
+    ports:
+      - "5434:5433"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U user -d broker_test -p 5433"]
+      interval: 2s
+      timeout: 3s
+      retries: 30
+
+  tests:
+    build:
+      context: .
+      dockerfile: Dockerfile.test
+    network_mode: "service:postgres"
+    depends_on:
+      postgres:
+        condition: service_healthy

docker-compose.yml

@@ -0,0 +1,50 @@
+services:
+  postgres:
+    image: docker.io/library/postgres:16-alpine
+    environment:
+      POSTGRES_DB: broker
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres -d broker"]
+      interval: 2s
+      timeout: 3s
+      retries: 30
+    restart: unless-stopped
+
+  # One-shot: applies migrations and creates/rotates the least-privilege
+  # broker_admin/broker_runtime/broker_enqueue roles, then exits. The
+  # broker service below only starts once this completes successfully.
+  migrate:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      postgres:
+        condition: service_healthy
+    volumes:
+      - ./broker.docker.yaml:/etc/pgsql-broker/broker.yaml:ro
+    environment:
+      PGUSER: postgres
+      PGPASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
+      BROKER_ADMIN_PASSWORD: ${BROKER_ADMIN_PASSWORD:?set BROKER_ADMIN_PASSWORD in .env}
+      BROKER_RUNTIME_PASSWORD: ${BROKER_RUNTIME_PASSWORD:?set BROKER_RUNTIME_PASSWORD in .env}
+      BROKER_ENQUEUE_PASSWORD: ${BROKER_ENQUEUE_PASSWORD:?set BROKER_ENQUEUE_PASSWORD in .env}
+    command: ["install", "--with-roles"]
+    restart: "no"
+
+  broker:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+    volumes:
+      - ./broker.docker.yaml:/etc/pgsql-broker/broker.yaml:ro
+    restart: unless-stopped
+
+volumes:
+  postgres-data:

go.mod

@@ -1,21 +1,31 @@
 module git.warky.dev/wdevs/pgsql-broker
 
-go 1.25.5
+go 1.26.0
 
 require (
 	github.com/lib/pq v1.10.9
+	github.com/prometheus/client_golang v1.20.5
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
+	golang.org/x/term v0.46.0
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/sagikazarmark/locafero v0.11.0 // indirect
 	github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
@@ -23,7 +33,7 @@ require (
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/sys v0.48.0 // indirect
 	golang.org/x/text v0.28.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 )

go.sum

@@ -1,3 +1,7 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -11,18 +15,32 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
 github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
@@ -45,12 +63,16 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
-golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
+golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
 golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
 golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/broker/adapter/database.go

@@ -19,6 +19,11 @@ type DBAdapter interface {
 	// Begin starts a new transaction
 	Begin(ctx context.Context) (DBTransaction, error)
 
+	// Conn returns a single physical connection pinned out of the pool, for
+	// session-scoped state (e.g. advisory locks) that must survive across
+	// calls. The caller owns it and must Close() it when done.
+	Conn(ctx context.Context) (*sql.Conn, error)
+
 	// Exec executes a query without returning rows
 	Exec(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
 

pkg/broker/adapter/postgres.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -22,6 +23,10 @@ type PostgresConfig struct {
 	MaxIdleConns    int
 	ConnMaxLifetime time.Duration
 	ConnMaxIdleTime time.Duration
+	// ApplicationName identifies this instance's pool connections in
+	// pg_stat_activity (e.g. "PGSQL_BROKER_INSTANCE1"). The LISTEN
+	// connection appends "_LISTENER" to this value.
+	ApplicationName string
 }
 
 // PostgresAdapter implements DBAdapter for PostgreSQL
@@ -170,7 +175,7 @@ func (p *PostgresAdapter) Query(ctx context.Context, query string, args ...inter
 
 // Listen starts listening on a PostgreSQL notification channel
 func (p *PostgresAdapter) Listen(ctx context.Context, channel string, handler NotificationHandler) error {
-	connStr := p.buildConnectionString()
+	connStr := p.buildConnectionStringWithAppName(p.config.ApplicationName + "_LISTENER")
 
 	reportProblem := func(ev pq.ListenerEventType, err error) {
 		if err != nil {
@@ -192,8 +197,11 @@ func (p *PostgresAdapter) Listen(ctx context.Context, channel string, handler No
 
 	p.logger.Info("listening on channel", "channel", channel)
 
-	// Start notification handler in goroutine
+	// Start notification handler in a supervised goroutine: it must keep
-	go func() {
+	// running for the life of the process, so a panic (e.g. from a
+	// misbehaving handler) is logged and the loop restarted rather than
+	// silently dying.
+	SupervisedGo(p.logger, "listener-"+channel, func() {
 		for {
 			select {
 			case n := <-listener.Notify:
@@ -208,10 +216,14 @@ func (p *PostgresAdapter) Listen(ctx context.Context, channel string, handler No
 				p.logger.Info("stopping listener", "channel", channel)
 				return
 			case <-time.After(90 * time.Second):
-				go listener.Ping()
+				SafeGo(p.logger, "listener-ping-"+channel, func() {
+					if err := listener.Ping(); err != nil {
+						p.logger.Error("listener ping failed", "channel", channel, "error", err)
+					}
+				})
 			}
 		}
-	}()
+	})
 
 	return nil
 }
@@ -229,24 +241,58 @@ func (p *PostgresAdapter) Unlisten(ctx context.Context, channel string) error {
 	return listener.Unlisten(channel)
 }
 
-// buildConnectionString builds a PostgreSQL connection string
+// buildConnectionString builds a PostgreSQL connection string for the
+// pooled connection, using the adapter's own application name.
 func (p *PostgresAdapter) buildConnectionString() string {
+	return p.buildConnectionStringWithAppName(p.config.ApplicationName)
+}
+
+// buildConnectionStringWithAppName builds a PostgreSQL connection string
+// with the given application_name, so pooled and LISTEN connections can be
+// told apart in pg_stat_activity.
+func (p *PostgresAdapter) buildConnectionStringWithAppName(appName string) string {
 	sslMode := p.config.SSLMode
 	if sslMode == "" {
 		sslMode = "disable"
 	}
 
 	return fmt.Sprintf(
-		"host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
+		"host=%s port=%d user=%s password=%s dbname=%s sslmode=%s application_name=%s options='-c search_path=broker,public'",
 		p.config.Host,
 		p.config.Port,
 		p.config.User,
 		p.config.Password,
 		p.config.Database,
 		sslMode,
+		quoteDSNValue(appName),
 	)
 }
 
+// quoteDSNValue escapes a value for use in a libpq keyword/value connection
+// string, single-quoting it and backslash-escaping embedded backslashes and
+// quotes per the libpq connection string format.
+func quoteDSNValue(v string) string {
+	v = strings.ReplaceAll(v, `\`, `\\`)
+	v = strings.ReplaceAll(v, `'`, `\'`)
+	return "'" + v + "'"
+}
+
+// Conn returns a single physical connection pinned out of the pool, for
+// session-scoped operations (e.g. pg_try_advisory_lock) that must survive
+// across calls and must not be silently reaped or handed to another caller
+// by the pool. The caller owns its lifecycle and must Close() it.
+func (p *PostgresAdapter) Conn(ctx context.Context) (*sql.Conn, error) {
+	p.mu.RLock()
+	db := p.db
+	p.mu.RUnlock()
+
+	if db == nil {
+		return nil, fmt.Errorf("database connection not established")
+	}
+
+	return db.Conn(ctx)
+}
+
 // postgresTransaction implements DBTransaction
 type postgresTransaction struct {
 	tx *sql.Tx

pkg/broker/adapter/safego.go

@@ -0,0 +1,56 @@
+package adapter
+
+import (
+	"runtime/debug"
+	"time"
+)
+
+// RecoverAndLog recovers a panic (if any) and logs it with a stack trace.
+// Call it via `defer adapter.RecoverAndLog(logger, "name")` at the top of
+// any goroutine body that must never be allowed to crash the process.
+func RecoverAndLog(logger Logger, name string) {
+	if r := recover(); r != nil {
+		logger.Error("recovered from panic", "component", name, "panic", r, "stack", string(debug.Stack()))
+	}
+}
+
+// SafeGo runs fn in a new goroutine, recovering any panic so it can never
+// crash the process. Use for one-shot/fire-and-forget goroutines; a panic is
+// logged (with its stack trace) and the goroutine simply ends.
+func SafeGo(logger Logger, name string, fn func()) {
+	go func() {
+		defer RecoverAndLog(logger, name)
+		fn()
+	}()
+}
+
+// SupervisedGo runs fn in a new goroutine. If fn panics, the panic is logged
+// and fn is restarted (after a short backoff) instead of letting the
+// goroutine die permanently. Use for long-running loops (ticker routines,
+// notification listeners, worker loops) that must keep running for the life
+// of the process. fn must return normally, without panicking, once its own
+// shutdown/context-done condition is met -- a clean return is not restarted.
+func SupervisedGo(logger Logger, name string, fn func()) {
+	go func() {
+		for {
+			if runSupervised(logger, name, fn) {
+				return
+			}
+			time.Sleep(time.Second)
+		}
+	}()
+}
+
+// runSupervised runs fn once, recovering a panic if it occurs. It returns
+// true if fn returned normally (no restart needed) and false if it panicked
+// (caller should restart it).
+func runSupervised(logger Logger, name string, fn func()) (clean bool) {
+	defer func() {
+		if r := recover(); r != nil {
+			logger.Error("recovered from panic, restarting", "component", name, "panic", r, "stack", string(debug.Stack()))
+			clean = false
+		}
+	}()
+	fn()
+	return true
+}

pkg/broker/broker.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/config"
+	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/metrics"
 )
 
 // Broker manages multiple database instances
@@ -17,6 +19,8 @@ type Broker struct {
 	instances []*DatabaseInstance
 	ctx       context.Context
 	cancel    context.CancelFunc
+	metrics   *metrics.Metrics
+	server    *metrics.Server
 	shutdown  bool
 	mu        sync.RWMutex
 }
@@ -33,6 +37,16 @@ func New(cfg *config.Config, logger adapter.Logger, version string) (*Broker, er
 		ctx:       ctx,
 		cancel:    cancel,
 	}
+	if cfg.Broker.MetricsEnabled {
+		broker.metrics = metrics.New()
+		broker.metrics.SetDatabaseCount(len(cfg.Databases))
+		addr := fmt.Sprintf("%s:%d", cfg.Broker.MetricsHost, cfg.Broker.MetricsPort)
+		server, err := metrics.NewServer(broker.metrics, addr, broker.logger)
+		if err != nil {
+			return nil, err
+		}
+		broker.server = server
+	}
 
 	return broker, nil
 }
@@ -41,15 +55,20 @@ func New(cfg *config.Config, logger adapter.Logger, version string) (*Broker, er
 func (b *Broker) Start() error {
 	b.logger.Info("starting broker", "database_count", len(b.config.Databases))
 
-	// Create and start an instance for each database
+	// Create and start an instance for each enabled database
-	for i, dbCfg := range b.config.Databases {
+	for i := range b.config.Databases {
+		dbCfg := &b.config.Databases[i]
+		if dbCfg.Disabled {
+			b.logger.Info("skipping disabled database", "name", dbCfg.Name)
+			continue
+		}
 		b.logger.Info("starting database instance", "name", dbCfg.Name, "host", dbCfg.Host, "database", dbCfg.Database)
 
 		// Create database adapter
 		dbAdapter := adapter.NewPostgresAdapter(dbCfg.ToPostgresConfig(), b.logger)
 
 		// Create database instance
-		instance, err := NewDatabaseInstance(b.config, &dbCfg, dbAdapter, b.logger, b.version, b.ctx)
+		instance, err := NewDatabaseInstance(b.config, dbCfg, dbAdapter, b.logger, b.version, b.ctx, b.metrics)
 		if err != nil {
 			// Stop any already-started instances
 			b.stopInstances()
@@ -67,7 +86,14 @@ func (b *Broker) Start() error {
 		b.logger.Info("database instance started", "name", dbCfg.Name, "instance_id", instance.ID)
 	}
 
+	if len(b.instances) == 0 {
+		return fmt.Errorf("no enabled databases configured (all %d database(s) are disabled)", len(b.config.Databases))
+	}
+
 	b.logger.Info("broker started successfully", "database_instances", len(b.instances))
+	if b.server != nil {
+		b.server.Start()
+	}
 	return nil
 }
 
@@ -88,6 +114,13 @@ func (b *Broker) Stop() error {
 
 	// Stop all instances
 	b.stopInstances()
+	if b.server != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := b.server.Stop(ctx); err != nil {
+			b.logger.Error("failed to stop metrics server", "error", err)
+		}
+	}
 
 	b.logger.Info("broker stopped")
 	return nil
@@ -100,6 +133,7 @@ func (b *Broker) stopInstances() {
 		wg.Add(1)
 		go func(inst *DatabaseInstance) {
 			defer wg.Done()
+			defer adapter.RecoverAndLog(b.logger, "stop-instance-"+inst.DatabaseName)
 			if err := inst.Stop(); err != nil {
 				b.logger.Error("failed to stop instance", "name", inst.DatabaseName, "error", err)
 			}

pkg/broker/config/config.go

@@ -2,10 +2,11 @@ package config
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
-	"github.com/spf13/viper"
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
+	"github.com/spf13/viper"
 )
 
 // Config holds all broker configuration
@@ -29,6 +30,19 @@ type DatabaseConfig struct {
 	ConnMaxLifetime time.Duration `mapstructure:"conn_max_lifetime"`
 	ConnMaxIdleTime time.Duration `mapstructure:"conn_max_idle_time"`
 	QueueCount      int           `mapstructure:"queue_count"`
+	// TenantID is the RLS tenant this instance's own workers operate as
+	// (via broker_set_tenant) when claiming/running jobs. Defaults to
+	// "default" so single-tenant deployments are unaffected.
+	TenantID string `mapstructure:"tenant_id"`
+	// AutoMigrate, when true, applies any pending embedded migrations on
+	// connect during normal `start`. When false (default), startup fails
+	// fast if the schema is behind, naming the missing migrations and
+	// pointing at `pgsql-broker install`.
+	AutoMigrate bool `mapstructure:"auto_migrate"`
+	// Disabled, when true, excludes this database from `start` (no
+	// instance is created for it). Defaults to false so existing config
+	// files without this field are unaffected.
+	Disabled bool `mapstructure:"disabled"`
 }
 
 // BrokerConfig holds broker-specific settings
@@ -38,8 +52,23 @@ type BrokerConfig struct {
 	QueueTimerSec        int           `mapstructure:"queue_timer_sec"`
 	QueueBufferSize      int           `mapstructure:"queue_buffer_size"`
 	WorkerIdleTimeoutSec int           `mapstructure:"worker_idle_timeout_sec"`
-	NotifyRetrySeconds   time.Duration `mapstructure:"notify_retry_seconds"`
+	NotifyRetryInterval  time.Duration `mapstructure:"notify_retry_seconds"`
 	EnableDebug          bool          `mapstructure:"enable_debug"`
+	// LeaseSeconds is how long a claimed job's lease is valid for before
+	// broker_recover_stale_jobs considers it abandoned.
+	LeaseSeconds int `mapstructure:"lease_seconds"`
+	// StaleJobRecoverySec is the interval between broker_recover_stale_jobs sweeps.
+	StaleJobRecoverySec int `mapstructure:"stale_job_recovery_sec"`
+	// MetricsEnabled controls whether the embedded Prometheus metrics HTTP
+	// server (exposition endpoint + HTML dashboard) is started.
+	MetricsEnabled bool `mapstructure:"metrics_enabled"`
+	// MetricsHost is the bind address for the metrics server.
+	MetricsHost string `mapstructure:"metrics_host"`
+	// MetricsPort is the bind port for the metrics server.
+	MetricsPort int `mapstructure:"metrics_port"`
+	// QueueDepthPollSec is the interval between polls of pending job counts
+	// used to populate the broker_jobs_queued gauge.
+	QueueDepthPollSec int `mapstructure:"queue_depth_poll_sec"`
 }
 
 // LoggingConfig holds logging settings
@@ -103,6 +132,12 @@ func setDefaults(v *viper.Viper) {
 	v.SetDefault("broker.worker_idle_timeout_sec", 10)
 	v.SetDefault("broker.notify_retry_seconds", 30*time.Second)
 	v.SetDefault("broker.enable_debug", false)
+	v.SetDefault("broker.lease_seconds", 60)
+	v.SetDefault("broker.stale_job_recovery_sec", 30)
+	v.SetDefault("broker.metrics_enabled", true)
+	v.SetDefault("broker.metrics_host", "0.0.0.0")
+	v.SetDefault("broker.metrics_port", 9469)
+	v.SetDefault("broker.queue_depth_poll_sec", 15)
 
 	// Logging defaults
 	v.SetDefault("logging.level", "info")
@@ -116,7 +151,8 @@ func validateConfig(config *Config) error {
 	}
 
 	// Validate each database configuration
-	for i, db := range config.Databases {
+	for i := range config.Databases {
+		db := &config.Databases[i]
 		if db.Name == "" {
 			return fmt.Errorf("database[%d]: name is required", i)
 		}
@@ -160,6 +196,9 @@ func applyDatabaseDefaults(config *Config) {
 		if db.QueueCount == 0 {
 			db.QueueCount = 4
 		}
+		if db.TenantID == "" {
+			db.TenantID = "default"
+		}
 	}
 }
 
@@ -176,5 +215,6 @@ func (d *DatabaseConfig) ToPostgresConfig() adapter.PostgresConfig {
 		MaxIdleConns:    d.MaxIdleConns,
 		ConnMaxLifetime: d.ConnMaxLifetime,
 		ConnMaxIdleTime: d.ConnMaxIdleTime,
+		ApplicationName: fmt.Sprintf("PGSQL_BROKER_%s", strings.ToUpper(d.Name)),
 	}
 }

pkg/broker/config/dbedit.go

@@ -0,0 +1,337 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"gopkg.in/yaml.v3"
+)
+
+// FileDoc wraps the parsed YAML node tree of a broker config file so that
+// database entries can be added, removed, or toggled while leaving the
+// rest of the file (formatting, comments, unrelated keys) intact.
+type FileDoc struct {
+	path string
+	root *yaml.Node
+}
+
+// databaseFileConfig mirrors DatabaseConfig with duration fields represented
+// as strings, which is the form used by the human-edited YAML config. YAML
+// unmarshalling does not parse strings into time.Duration automatically.
+type databaseFileConfig struct {
+	Name            string `yaml:"name"`
+	Host            string `yaml:"host"`
+	Port            int    `yaml:"port"`
+	Database        string `yaml:"database"`
+	User            string `yaml:"user"`
+	Password        string `yaml:"password"`
+	SSLMode         string `yaml:"sslmode"`
+	MaxOpenConns    int    `yaml:"max_open_conns"`
+	MaxIdleConns    int    `yaml:"max_idle_conns"`
+	ConnMaxLifetime string `yaml:"conn_max_lifetime"`
+	ConnMaxIdleTime string `yaml:"conn_max_idle_time"`
+	QueueCount      int    `yaml:"queue_count"`
+	TenantID        string `yaml:"tenant_id"`
+	AutoMigrate     bool   `yaml:"auto_migrate"`
+	Disabled        bool   `yaml:"disabled"`
+}
+
+func decodeDatabase(node *yaml.Node) (DatabaseConfig, error) {
+	var raw databaseFileConfig
+	if err := node.Decode(&raw); err != nil {
+		return DatabaseConfig{}, err
+	}
+	db := DatabaseConfig{
+		Name: raw.Name, Host: raw.Host, Port: raw.Port, Database: raw.Database,
+		User: raw.User, Password: raw.Password, SSLMode: raw.SSLMode,
+		MaxOpenConns: raw.MaxOpenConns, MaxIdleConns: raw.MaxIdleConns,
+		QueueCount: raw.QueueCount, TenantID: raw.TenantID,
+		AutoMigrate: raw.AutoMigrate, Disabled: raw.Disabled,
+	}
+	var err error
+	if raw.ConnMaxLifetime != "" {
+		db.ConnMaxLifetime, err = time.ParseDuration(raw.ConnMaxLifetime)
+		if err != nil {
+			return DatabaseConfig{}, fmt.Errorf("conn_max_lifetime: %w", err)
+		}
+	}
+	if raw.ConnMaxIdleTime != "" {
+		db.ConnMaxIdleTime, err = time.ParseDuration(raw.ConnMaxIdleTime)
+		if err != nil {
+			return DatabaseConfig{}, fmt.Errorf("conn_max_idle_time: %w", err)
+		}
+	}
+	return db, nil
+}
+
+// LoadFileDoc reads and parses the YAML config file at path for editing.
+// A missing file is treated as an empty document so `db add` can be used
+// to create a new config file from scratch.
+func LoadFileDoc(path string) (*FileDoc, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, fmt.Errorf("failed to read config file %s: %w", path, err)
+		}
+		data = nil
+	}
+
+	var root yaml.Node
+	if len(data) > 0 {
+		if err := yaml.Unmarshal(data, &root); err != nil {
+			return nil, fmt.Errorf("failed to parse config file %s: %w", path, err)
+		}
+	}
+	if root.Kind == 0 {
+		root.Kind = yaml.DocumentNode
+		root.Content = []*yaml.Node{{Kind: yaml.MappingNode, Tag: "!!map"}}
+	}
+
+	return &FileDoc{path: path, root: &root}, nil
+}
+
+// Save writes the document back to its original path.
+func (f *FileDoc) Save() error {
+	data, err := yaml.Marshal(f.root)
+	if err != nil {
+		return fmt.Errorf("failed to marshal config: %w", err)
+	}
+	if err := os.WriteFile(f.path, data, 0o644); err != nil {
+		return fmt.Errorf("failed to write config file %s: %w", f.path, err)
+	}
+	return nil
+}
+
+func (f *FileDoc) mappingRoot() (*yaml.Node, error) {
+	if len(f.root.Content) == 0 {
+		return nil, fmt.Errorf("config file is empty")
+	}
+	m := f.root.Content[0]
+	if m.Kind != yaml.MappingNode {
+		return nil, fmt.Errorf("config file root is not a mapping")
+	}
+	return m, nil
+}
+
+// databasesSeq returns the "databases" sequence node, creating it (and the
+// key) if it isn't already present.
+func (f *FileDoc) databasesSeq() (*yaml.Node, error) {
+	m, err := f.mappingRoot()
+	if err != nil {
+		return nil, err
+	}
+	for i := 0; i+1 < len(m.Content); i += 2 {
+		if m.Content[i].Value == "databases" {
+			seq := m.Content[i+1]
+			if seq.Kind != yaml.SequenceNode {
+				return nil, fmt.Errorf("'databases' key in config is not a list")
+			}
+			return seq, nil
+		}
+	}
+	keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: "databases"}
+	seqNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	m.Content = append(m.Content, keyNode, seqNode)
+	return seqNode, nil
+}
+
+// ListDatabases decodes all database entries currently in the document, in
+// file order.
+func (f *FileDoc) ListDatabases() ([]DatabaseConfig, error) {
+	seq, err := f.databasesSeq()
+	if err != nil {
+		return nil, err
+	}
+	dbs := make([]DatabaseConfig, 0, len(seq.Content))
+	for _, item := range seq.Content {
+		db, err := decodeDatabase(item)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode database entry: %w", err)
+		}
+		dbs = append(dbs, db)
+	}
+	return dbs, nil
+}
+
+// FindDatabase returns the decoded entry with the given name.
+func (f *FileDoc) FindDatabase(name string) (DatabaseConfig, bool, error) {
+	dbs, err := f.ListDatabases()
+	if err != nil {
+		return DatabaseConfig{}, false, err
+	}
+	for i := range dbs {
+		db := &dbs[i]
+		if db.Name == name {
+			return *db, true, nil
+		}
+	}
+	return DatabaseConfig{}, false, nil
+}
+
+// LastDatabase returns the last database entry in the file, used to prime
+// defaults for a new `db add` when no --from instance is given.
+func (f *FileDoc) LastDatabase() (DatabaseConfig, bool, error) {
+	dbs, err := f.ListDatabases()
+	if err != nil {
+		return DatabaseConfig{}, false, err
+	}
+	if len(dbs) == 0 {
+		return DatabaseConfig{}, false, nil
+	}
+	return dbs[len(dbs)-1], true, nil
+}
+
+// AddDatabase appends a new database entry. It fails if an entry with the
+// same name already exists.
+func (f *FileDoc) AddDatabase(db DatabaseConfig) error {
+	seq, err := f.databasesSeq()
+	if err != nil {
+		return err
+	}
+	for _, item := range seq.Content {
+		existing, err := decodeDatabase(item)
+		if err != nil {
+			return fmt.Errorf("failed to decode database entry: %w", err)
+		}
+		if existing.Name == db.Name {
+			return fmt.Errorf("a database named %q already exists", db.Name)
+		}
+	}
+	seq.Content = append(seq.Content, dbToNode(db))
+	return nil
+}
+
+// RemoveDatabase deletes the entry with the given name, reporting whether
+// it was found.
+func (f *FileDoc) RemoveDatabase(name string) (bool, error) {
+	seq, err := f.databasesSeq()
+	if err != nil {
+		return false, err
+	}
+	for i, item := range seq.Content {
+		existing, err := decodeDatabase(item)
+		if err != nil {
+			return false, fmt.Errorf("failed to decode database entry: %w", err)
+		}
+		if existing.Name == name {
+			seq.Content = append(seq.Content[:i], seq.Content[i+1:]...)
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// SetDisabled sets (or clears) the disabled flag on the named entry,
+// reporting whether the entry was found.
+func (f *FileDoc) SetDisabled(name string, disabled bool) (bool, error) {
+	seq, err := f.databasesSeq()
+	if err != nil {
+		return false, err
+	}
+	for _, item := range seq.Content {
+		existing, err := decodeDatabase(item)
+		if err != nil {
+			return false, fmt.Errorf("failed to decode database entry: %w", err)
+		}
+		if existing.Name != name {
+			continue
+		}
+		if disabled {
+			setMappingKey(item, "disabled", &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!bool", Value: "true"})
+		} else {
+			removeMappingKey(item, "disabled")
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
+// setMappingKey sets key to value within a mapping node, appending the
+// pair if the key isn't already present.
+func setMappingKey(m *yaml.Node, key string, value *yaml.Node) {
+	for i := 0; i+1 < len(m.Content); i += 2 {
+		if m.Content[i].Value == key {
+			m.Content[i+1] = value
+			return
+		}
+	}
+	m.Content = append(m.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: key}, value)
+}
+
+// removeMappingKey removes key from a mapping node if present.
+func removeMappingKey(m *yaml.Node, key string) {
+	for i := 0; i+1 < len(m.Content); i += 2 {
+		if m.Content[i].Value == key {
+			m.Content = append(m.Content[:i], m.Content[i+2:]...)
+			return
+		}
+	}
+}
+
+// dbToNode builds a YAML mapping node for a database entry, omitting
+// fields left at their zero value so the on-disk defaults from
+// applyDatabaseDefaults keep applying (matching broker.example.yaml
+// style). Field order mirrors DatabaseConfig / broker.example.yaml.
+func dbToNode(db DatabaseConfig) *yaml.Node {
+	m := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+
+	add := func(key, value string) {
+		m.Content = append(m.Content,
+			&yaml.Node{Kind: yaml.ScalarNode, Value: key},
+			&yaml.Node{Kind: yaml.ScalarNode, Value: value},
+		)
+	}
+	addBool := func(key string, value bool) {
+		m.Content = append(m.Content,
+			&yaml.Node{Kind: yaml.ScalarNode, Value: key},
+			&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!bool", Value: fmt.Sprintf("%t", value)},
+		)
+	}
+	addInt := func(key string, value int) {
+		m.Content = append(m.Content,
+			&yaml.Node{Kind: yaml.ScalarNode, Value: key},
+			&yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%d", value)},
+		)
+	}
+
+	add("name", db.Name)
+	add("host", db.Host)
+	if db.Port != 0 {
+		addInt("port", db.Port)
+	}
+	add("database", db.Database)
+	add("user", db.User)
+	if db.Password != "" {
+		add("password", db.Password)
+	}
+	if db.SSLMode != "" {
+		add("sslmode", db.SSLMode)
+	}
+	if db.MaxOpenConns != 0 {
+		addInt("max_open_conns", db.MaxOpenConns)
+	}
+	if db.MaxIdleConns != 0 {
+		addInt("max_idle_conns", db.MaxIdleConns)
+	}
+	if db.ConnMaxLifetime != 0 {
+		add("conn_max_lifetime", db.ConnMaxLifetime.String())
+	}
+	if db.ConnMaxIdleTime != 0 {
+		add("conn_max_idle_time", db.ConnMaxIdleTime.String())
+	}
+	if db.QueueCount != 0 {
+		addInt("queue_count", db.QueueCount)
+	}
+	if db.TenantID != "" {
+		add("tenant_id", db.TenantID)
+	}
+	if db.AutoMigrate {
+		addBool("auto_migrate", db.AutoMigrate)
+	}
+	if db.Disabled {
+		addBool("disabled", db.Disabled)
+	}
+
+	return m
+}

pkg/broker/config/dbedit_test.go

@@ -0,0 +1,102 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileDocDatabaseManagement(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "broker.yaml")
+	require.NoError(t, os.WriteFile(path, []byte(`broker:
+  name: test
+# keep this comment
+databases:
+  - name: primary
+    host: db.example
+    port: 5433
+    database: jobs
+    user: broker
+    password: secret
+    sslmode: verify-full
+    max_open_conns: 12
+    max_idle_conns: 4
+    conn_max_lifetime: 2m
+    conn_max_idle_time: 30s
+    queue_count: 3
+    tenant_id: tenant-a
+    auto_migrate: true
+`), 0o600))
+
+	doc, err := LoadFileDoc(path)
+	require.NoError(t, err)
+	primary, found, err := doc.FindDatabase("primary")
+	require.NoError(t, err)
+	require.True(t, found)
+	require.Equal(t, 2*time.Minute, primary.ConnMaxLifetime)
+	require.Equal(t, 30*time.Second, primary.ConnMaxIdleTime)
+	require.True(t, primary.AutoMigrate)
+
+	copy := primary
+	copy.Name = "replica"
+	require.NoError(t, doc.AddDatabase(copy))
+	found, err = doc.SetDisabled("replica", true)
+	require.NoError(t, err)
+	require.True(t, found)
+	require.NoError(t, doc.Save())
+
+	doc, err = LoadFileDoc(path)
+	require.NoError(t, err)
+	replica, found, err := doc.FindDatabase("replica")
+	require.NoError(t, err)
+	require.True(t, found)
+	require.True(t, replica.Disabled)
+	require.NoError(t, func() error {
+		found, err := doc.SetDisabled("replica", false)
+		require.True(t, found)
+		return err
+	}())
+	require.NoError(t, doc.Save())
+
+	doc, err = LoadFileDoc(path)
+	require.NoError(t, err)
+	replica, found, err = doc.FindDatabase("replica")
+	require.NoError(t, err)
+	require.True(t, found)
+	require.False(t, replica.Disabled)
+
+	removed, err := doc.RemoveDatabase("primary")
+	require.NoError(t, err)
+	require.True(t, removed)
+	require.NoError(t, doc.Save())
+	contents, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.Contains(t, string(contents), "# keep this comment")
+
+	_, found, err = doc.FindDatabase("primary")
+	require.NoError(t, err)
+	require.False(t, found)
+}
+
+func TestFileDocMissingFileCanAddDatabase(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "new.yaml")
+	doc, err := LoadFileDoc(path)
+	require.NoError(t, err)
+	require.NoError(t, doc.AddDatabase(DatabaseConfig{
+		Name:     "new",
+		Host:     "localhost",
+		Database: "jobs",
+		User:     "broker",
+	}))
+	require.NoError(t, doc.Save())
+
+	reloaded, err := LoadFileDoc(path)
+	require.NoError(t, err)
+	dbs, err := reloaded.ListDatabases()
+	require.NoError(t, err)
+	require.Len(t, dbs, 1)
+	require.Equal(t, "new", dbs[0].Name)
+}

pkg/broker/database_instance.go

@@ -2,15 +2,18 @@ package broker
 
 import (
 	"context"
-	"database/sql" // Import sql package
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/config"
+	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/install"
+	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/metrics"
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/models"
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/queue"
 )
@@ -34,15 +37,26 @@ type DatabaseInstance struct {
 	shutdownMu   sync.RWMutex
 	jobsHandled  int64
 	startTime    time.Time
+	metrics      *metrics.Metrics
+
+	// sessionConn holds the pg_try_advisory_lock acquired by
+	// registerInstance. The lock is scoped to this one physical connection,
+	// so it must be kept open (never returned to the pool) for the life of
+	// the process and explicitly unlocked on Stop().
+	sessionConn *sql.Conn
 }
 
 // NewDatabaseInstance creates a new database instance
-func NewDatabaseInstance(cfg *config.Config, dbCfg *config.DatabaseConfig, db adapter.DBAdapter, logger adapter.Logger, version string, parentCtx context.Context) (*DatabaseInstance, error) {
+func NewDatabaseInstance(cfg *config.Config, dbCfg *config.DatabaseConfig, db adapter.DBAdapter, logger adapter.Logger, version string, parentCtx context.Context, brokerMetrics ...*metrics.Metrics) (*DatabaseInstance, error) {
 	hostname, err := os.Hostname()
 	if err != nil {
 		hostname = "unknown"
 	}
 
+	var instanceMetrics *metrics.Metrics
+	if len(brokerMetrics) > 0 {
+		instanceMetrics = brokerMetrics[0]
+	}
 	instance := &DatabaseInstance{
 		Name:         fmt.Sprintf("%s-%s", cfg.Broker.Name, dbCfg.Name),
 		DatabaseName: dbCfg.Name,
@@ -56,6 +70,7 @@ func NewDatabaseInstance(cfg *config.Config, dbCfg *config.DatabaseConfig, db ad
 		queues:       make(map[int]*queue.Queue),
 		ctx:          parentCtx,
 		startTime:    time.Now(),
+		metrics:      instanceMetrics,
 	}
 
 	return instance, nil
@@ -70,6 +85,11 @@ func (i *DatabaseInstance) Start() error {
 		return fmt.Errorf("failed to connect to database: %w", err)
 	}
 
+	// Ensure the schema is up to date before touching any broker objects.
+	if err := i.ensureSchema(); err != nil {
+		return err
+	}
+
 	// Register instance in database
 	if err := i.registerInstance(); err != nil {
 		return fmt.Errorf("failed to register instance: %w", err)
@@ -87,10 +107,90 @@ func (i *DatabaseInstance) Start() error {
 		return fmt.Errorf("failed to start listener: %w", err)
 	}
 
-	// Start ping routine
+	// Start ping routine (auto-restarted on panic; must run for the life of
-	go i.pingRoutine()
+	// the process)
+	adapter.SupervisedGo(i.logger, "ping-routine", i.pingRoutine)
+
+	// Start stale/expired-lease job recovery routine (auto-restarted on
+	// panic; must run for the life of the process)
+	adapter.SupervisedGo(i.logger, "stale-job-recovery-routine", i.staleJobRecoveryRoutine)
 
 	i.logger.Info("database instance started successfully")
+	if i.metrics != nil {
+		adapter.SupervisedGo(i.logger, "metrics-queue-depth-routine", i.queueDepthRoutine)
+	}
+	return nil
+}
+
+// queueDepthRoutine periodically exports pending jobs grouped by queue.
+func (i *DatabaseInstance) queueDepthRoutine() {
+	interval := time.Duration(i.config.Broker.QueueDepthPollSec) * time.Second
+	if interval <= 0 {
+		interval = 15 * time.Second
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	i.updateQueueDepthMetrics()
+	for {
+		select {
+		case <-ticker.C:
+			i.updateQueueDepthMetrics()
+		case <-i.ctx.Done():
+			return
+		}
+	}
+}
+
+func (i *DatabaseInstance) updateQueueDepthMetrics() {
+	rows, err := i.db.Query(i.ctx, "SELECT job_queue, COUNT(*) FROM broker.broker_jobs WHERE complete_status = 0 GROUP BY job_queue")
+	if err != nil {
+		i.logger.Warn("failed to collect queue depth metrics", "error", err)
+		return
+	}
+	defer rows.Close()
+	for queueNumber := 1; queueNumber <= i.dbConfig.QueueCount; queueNumber++ {
+		i.metrics.SetJobsQueued(i.DatabaseName, queueNumber, 0)
+	}
+	for rows.Next() {
+		var queueNumber, count int
+		if err := rows.Scan(&queueNumber, &count); err != nil {
+			i.logger.Warn("failed to scan queue depth metric", "error", err)
+			return
+		}
+		i.metrics.SetJobsQueued(i.DatabaseName, queueNumber, float64(count))
+	}
+	if err := rows.Err(); err != nil {
+		i.logger.Warn("failed to read queue depth metrics", "error", err)
+	}
+}
+
+// ensureSchema checks the embedded migration set against the database and,
+// depending on dbConfig.AutoMigrate, either applies pending migrations or
+// fails startup fast rather than running against a stale/missing schema.
+func (i *DatabaseInstance) ensureSchema() error {
+	installer := install.New(i.db, i.logger)
+
+	pending, err := installer.PendingMigrations(i.ctx)
+	if err != nil {
+		return fmt.Errorf("failed to check schema migrations: %w", err)
+	}
+
+	if len(pending) == 0 {
+		return nil
+	}
+
+	if !i.dbConfig.AutoMigrate {
+		return fmt.Errorf(
+			"schema is missing or behind: %d migration(s) not applied (%s); either run `pgsql-broker install` or set databases[].auto_migrate: true",
+			len(pending), strings.Join(pending, ", "),
+		)
+	}
+
+	i.logger.Info("auto-migrating database schema", "pending", pending)
+	if err := installer.ApplyMigrations(i.ctx); err != nil {
+		return fmt.Errorf("auto-migration failed: %w", err)
+	}
+
 	return nil
 }
 
@@ -116,10 +216,26 @@ func (i *DatabaseInstance) Stop() error {
 	}
 	i.queuesMu.Unlock()
 
-	// Update instance status in database
+	// Update instance status in database. i.ctx may already be canceled by
-	if err := i.shutdownInstance(); err != nil {
+	// the parent broker's Stop(), so use a fresh short-lived context here.
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	if err := i.shutdownInstance(shutdownCtx); err != nil {
 		i.logger.Error("failed to shutdown instance in database", "error", err)
 	}
+	cancel()
+
+	// Release the advisory lock and close the pinned session connection.
+	if i.sessionConn != nil {
+		if _, err := i.sessionConn.ExecContext(context.Background(),
+			"SELECT pg_advisory_unlock(hashtextextended($1, 0))", "broker:"+i.Name,
+		); err != nil {
+			i.logger.Error("failed to release advisory lock", "error", err)
+		}
+		if err := i.sessionConn.Close(); err != nil {
+			i.logger.Error("failed to close session connection", "error", err)
+		}
+		i.sessionConn = nil
+	}
 
 	// Close database connection
 	if err := i.db.Close(); err != nil {
@@ -130,72 +246,48 @@ func (i *DatabaseInstance) Stop() error {
 	return nil
 }
 
-// registerInstance registers the instance in the database
+// registerInstance registers the instance in the database. The advisory
+// lock taken by broker_register_instance is session-scoped, so this runs on
+// a connection pinned out of the pool (i.sessionConn) that is kept open for
+// the life of the process rather than returned after this call.
 func (i *DatabaseInstance) registerInstance() error {
+	conn, err := i.db.Conn(i.ctx)
+	if err != nil {
+		return fmt.Errorf("failed to acquire session connection: %w", err)
+	}
+
 	var retval int
 	var errmsg string
-	var nullableInstanceID sql.NullInt64 // Change to nullable type
+	var nullableInstanceID sql.NullInt64
 
 	i.logger.Debug("registering instance", "name", i.Name, "hostname", i.Hostname, "pid", i.PID, "version", i.Version, "queue_count", i.dbConfig.QueueCount)
-	err := i.db.QueryRow(i.ctx,
+	err = conn.QueryRowContext(i.ctx,
-		"SELECT p_retval, p_errmsg, p_instance_id FROM broker_register_instance($1, $2, $3, $4, $5)",
+		"SELECT p_retval, p_errmsg, p_instance_id FROM broker.broker_register_instance($1, $2, $3, $4, $5)",
 		i.Name, i.Hostname, i.PID, i.Version, i.dbConfig.QueueCount,
 	).Scan(&retval, &errmsg, &nullableInstanceID)
 
 	if err != nil {
+		conn.Close()
 		i.logger.Error("query error during instance registration", "error", err)
 		return fmt.Errorf("query error: %w", err)
 	}
 
-	if retval == 3 {
+	if retval > 0 {
-		i.logger.Warn("another broker instance is already active, attempting to retrieve ID", "error", errmsg)
+		conn.Close()
-		// Try to retrieve the ID of the active instance
-		var activeID int64
-		err := i.db.QueryRow(i.ctx,
-			"SELECT id_broker_queueinstance FROM broker_queueinstance WHERE name = $1 AND hostname = $2 AND status = 'active' ORDER BY started_at DESC LIMIT 1",
-			i.Name, i.Hostname,
-		).Scan(&activeID)
-		if err != nil {
-			i.logger.Error("failed to retrieve ID of active instance", "error", err)
-			return fmt.Errorf("failed to retrieve ID of active instance: %w", err)
-		}
-		i.ID = activeID
-		i.logger.Info("retrieved active instance ID", "id", i.ID)
-		return nil
-	} else if retval > 0 {
 		i.logger.Error("broker_register_instance error", "retval", retval, "errmsg", errmsg)
 		return fmt.Errorf("broker_register_instance error: %s", errmsg)
 	}
 
-	// If successfully registered, nullableInstanceID.Valid will be true
+	if !nullableInstanceID.Valid {
-	if nullableInstanceID.Valid {
+		conn.Close()
-		i.ID = nullableInstanceID.Int64
-		i.logger.Info("registered new instance", "id", i.ID)
-
-	// Debug logging: Retrieve all entries from broker_queueinstance
-	rows, err := i.db.Query(i.ctx, "SELECT id_broker_queueinstance, name, hostname, status FROM broker_queueinstance")
-	if err != nil {
-		i.logger.Error("debug query failed", "error", err)
-	} else {
-		defer rows.Close()
-		for rows.Next() {
-			var id int64
-			var name, hostname, status string
-			if err := rows.Scan(&id, &name, &hostname, &status); err != nil {
-				i.logger.Error("debug scan failed", "error", err)
-				break
-			}
-			i.logger.Debug("broker_queueinstance entry", "id", id, "name", name, "hostname", hostname, "status", status)
-		}
-	}
-	} else {
-		// This case should ideally not happen if retval is 0 (success)
-		// but if it does, it means p_instance_id was NULL despite success.
-		// This would be an unexpected scenario.
 		i.logger.Error("broker_register_instance returned success but no instance ID", "retval", retval, "errmsg", errmsg)
 		return fmt.Errorf("broker_register_instance returned success but no instance ID")
 	}
 
+	i.ID = nullableInstanceID.Int64
+	i.sessionConn = conn
+	i.logger.Info("registered new instance", "id", i.ID)
+
 	return nil
 }
 
@@ -204,6 +296,11 @@ func (i *DatabaseInstance) startQueues() error {
 	i.queuesMu.Lock()
 	defer i.queuesMu.Unlock()
 
+	leaseSeconds := i.config.Broker.LeaseSeconds
+	if leaseSeconds <= 0 {
+		leaseSeconds = 60
+	}
+
 	for queueNum := 1; queueNum <= i.dbConfig.QueueCount; queueNum++ {
 		queueCfg := queue.Config{
 			Number:       queueNum,
@@ -214,6 +311,10 @@ func (i *DatabaseInstance) startQueues() error {
 			BufferSize:   i.config.Broker.QueueBufferSize,
 			TimerSeconds: i.config.Broker.QueueTimerSec,
 			FetchSize:    i.config.Broker.FetchQueryQueSize,
+			TenantID:     i.dbConfig.TenantID,
+			LeaseSeconds: leaseSeconds,
+			Metrics:      i.metrics,
+			DatabaseName: i.DatabaseName,
 		}
 
 		q := queue.New(queueCfg)
@@ -224,6 +325,7 @@ func (i *DatabaseInstance) startQueues() error {
 		i.queues[queueNum] = q
 		i.logger.Info("queue started", "number", queueNum)
 	}
+	i.metrics.SetQueueCount(i.DatabaseName, len(i.queues))
 
 	return nil
 }
@@ -241,42 +343,38 @@ func (i *DatabaseInstance) startListener() error {
 	return nil
 }
 
-// handleNotification processes incoming job notifications
+// handleNotification processes incoming wake-up notifications. The payload
+// only carries the queue number (plus a job id kept for logging) --
+// NOTIFY is wake-only, never a hand-off of a job to execute directly, so the
+// woken worker always re-claims via broker_get.
 func (i *DatabaseInstance) handleNotification(n *adapter.Notification) {
+	defer adapter.RecoverAndLog(i.logger, "handle-notification")
+
 	if i.config.Broker.EnableDebug {
 		i.logger.Debug("received notification", "channel", n.Channel, "payload", n.Payload)
 	}
 
-	var job models.Job
+	var wake models.WakeNotification
-	if err := json.Unmarshal([]byte(n.Payload), &job); err != nil {
+	if err := json.Unmarshal([]byte(n.Payload), &wake); err != nil {
 		i.logger.Error("failed to unmarshal notification", "error", err, "payload", n.Payload)
 		return
 	}
 
-	if job.ID <= 0 {
+	if wake.Queue <= 0 {
-		i.logger.Warn("notification missing job ID", "payload", n.Payload)
+		i.logger.Warn("notification missing queue number", "payload", n.Payload)
 		return
 	}
 
-	if job.JobQueue <= 0 {
-		i.logger.Warn("notification missing queue number", "job_id", job.ID)
-		return
-	}
-
-	// Get the queue
 	i.queuesMu.RLock()
-	q, exists := i.queues[job.JobQueue]
+	q, exists := i.queues[wake.Queue]
 	i.queuesMu.RUnlock()
 
 	if !exists {
-		i.logger.Warn("queue not found for job", "job_id", job.ID, "queue", job.JobQueue)
+		i.logger.Warn("queue not found for notification", "queue", wake.Queue, "job_id", wake.JobID)
 		return
 	}
 
-	// Add job to queue
+	q.Wake()
-	if err := q.AddJob(job); err != nil {
-		i.logger.Error("failed to add job to queue", "job_id", job.ID, "queue", job.JobQueue, "error", err)
-	}
 }
 
 // pingRoutine periodically updates the instance status in the database
@@ -304,13 +402,67 @@ func (i *DatabaseInstance) pingRoutine() {
 	}
 }
 
+// staleJobRecoveryRoutine periodically requeues (or dead-letters) jobs whose
+// lease has expired while still running.
+func (i *DatabaseInstance) staleJobRecoveryRoutine() {
+	interval := time.Duration(i.config.Broker.StaleJobRecoverySec) * time.Second
+	if interval <= 0 {
+		interval = 30 * time.Second
+	}
+
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			i.shutdownMu.RLock()
+			if i.shutdown {
+				i.shutdownMu.RUnlock()
+				return
+			}
+			i.shutdownMu.RUnlock()
+
+			if err := i.recoverStaleJobs(); err != nil {
+				i.logger.Error("stale job recovery failed", "error", err)
+			}
+
+		case <-i.ctx.Done():
+			return
+		}
+	}
+}
+
+// recoverStaleJobs invokes broker_recover_stale_jobs.
+func (i *DatabaseInstance) recoverStaleJobs() error {
+	var retval int
+	var errmsg string
+	var recoveredCount int
+
+	err := i.db.QueryRow(i.ctx, "SELECT p_retval, p_errmsg, p_recovered_count FROM broker.broker_recover_stale_jobs()").
+		Scan(&retval, &errmsg, &recoveredCount)
+	if err != nil {
+		return fmt.Errorf("query error: %w", err)
+	}
+
+	if retval > 0 {
+		return fmt.Errorf("broker_recover_stale_jobs error: %s", errmsg)
+	}
+
+	if recoveredCount > 0 {
+		i.logger.Info("recovered stale jobs", "count", recoveredCount)
+	}
+
+	return nil
+}
+
 // ping updates the instance ping timestamp
 func (i *DatabaseInstance) ping() error {
 	var retval int
 	var errmsg string
 
 	err := i.db.QueryRow(i.ctx,
-		"SELECT p_retval, p_errmsg FROM broker_ping_instance($1, $2)",
+		"SELECT p_retval, p_errmsg FROM broker.broker_ping_instance($1, $2)",
 		i.ID, i.jobsHandled,
 	).Scan(&retval, &errmsg)
 
@@ -326,12 +478,12 @@ func (i *DatabaseInstance) ping() error {
 }
 
 // shutdownInstance marks the instance as shutdown in the database
-func (i *DatabaseInstance) shutdownInstance() error {
+func (i *DatabaseInstance) shutdownInstance(ctx context.Context) error {
 	var retval int
 	var errmsg string
 
-	err := i.db.QueryRow(i.ctx,
+	err := i.db.QueryRow(ctx,
-		"SELECT p_retval, p_errmsg FROM broker_shutdown_instance($1)",
+		"SELECT p_retval, p_errmsg FROM broker.broker_shutdown_instance($1)",
 		i.ID,
 	).Scan(&retval, &errmsg)
 

pkg/broker/install/install.go

@@ -4,17 +4,47 @@ import (
 	"context"
 	"embed"
 	"fmt"
-	"io/fs"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 
+	"github.com/lib/pq"
+
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
 )
 
-//go:embed all:sql
+//go:embed all:sql/migrations
-var sqlFS embed.FS
+var migrationsFS embed.FS
 
-// Installer handles database schema installation
+//go:embed all:sql/roles
+var rolesFS embed.FS
+
+const migrationsDir = "sql/migrations"
+const rolesDir = "sql/roles"
+
+// migrationsTableSQL creates the version-tracking table itself. It is applied
+// unconditionally (idempotently) before any numbered migration file, and is
+// not itself a numbered migration.
+const migrationsTableSQL = `
+CREATE SCHEMA IF NOT EXISTS broker;
+CREATE TABLE IF NOT EXISTS broker.broker_schema_migrations (
+    version    BIGINT PRIMARY KEY,
+    name       TEXT NOT NULL,
+    applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+`
+
+var migrationFileRe = regexp.MustCompile(`^(\d+)_(.+)\.sql$`)
+
+// migrationFile describes one embedded migration.
+type migrationFile struct {
+	version int64
+	name    string
+	path    string
+}
+
+// Installer handles database schema installation via versioned migrations.
 type Installer struct {
 	db     adapter.DBAdapter
 	logger adapter.Logger
@@ -28,217 +58,423 @@ func New(db adapter.DBAdapter, logger adapter.Logger) *Installer {
 	}
 }
 
-// InstallSchema installs the complete database schema
+// loadMigrations reads and sorts every embedded migration file by numeric prefix.
-func (i *Installer) InstallSchema(ctx context.Context) error {
+func loadMigrations() ([]migrationFile, error) {
-	i.logger.Info("starting schema installation")
+	entries, err := migrationsFS.ReadDir(migrationsDir)
-
+	if err != nil {
-	// Install tables first
+		return nil, fmt.Errorf("failed to read migrations directory: %w", err)
-	if err := i.installTables(ctx); err != nil {
-		return fmt.Errorf("failed to install tables: %w", err)
 	}
 
-	// Then install procedures
+	var migrations []migrationFile
-	if err := i.installProcedures(ctx); err != nil {
+	for _, e := range entries {
-		return fmt.Errorf("failed to install procedures: %w", err)
+		if e.IsDir() {
+			continue
+		}
+		m := migrationFileRe.FindStringSubmatch(e.Name())
+		if m == nil {
+			continue
+		}
+		version, err := strconv.ParseInt(m[1], 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("invalid migration filename %s: %w", e.Name(), err)
+		}
+		migrations = append(migrations, migrationFile{
+			version: version,
+			name:    m[2],
+			path:    migrationsDir + "/" + e.Name(),
+		})
 	}
 
-	i.logger.Info("schema installation completed successfully")
+	sort.Slice(migrations, func(i, j int) bool { return migrations[i].version < migrations[j].version })
+	return migrations, nil
+}
+
+// ensureMigrationsTable creates the broker schema and the migrations
+// tracking table if they don't already exist. This is DDL and requires
+// CREATE privilege on the database -- only ApplyMigrations (run by an
+// admin-privileged connection, e.g. `pgsql-broker install`) calls it.
+func (i *Installer) ensureMigrationsTable(ctx context.Context) error {
+	if _, err := i.db.Exec(ctx, migrationsTableSQL); err != nil {
+		return fmt.Errorf("failed to ensure migrations table: %w", err)
+	}
 	return nil
 }
 
-// installTables installs all table definitions
+// migrationsTableExists reports whether the migrations tracking table is
-func (i *Installer) installTables(ctx context.Context) error {
+// present, without creating it -- a read-only check safe to run with a
-	i.logger.Info("installing tables")
+// least-privilege runtime role (e.g. broker_runtime) that has no CREATE
-
+// privilege on the database.
-	files, err := sqlFS.ReadDir("sql/tables")
+func (i *Installer) migrationsTableExists(ctx context.Context) (bool, error) {
+	var exists bool
+	err := i.db.QueryRow(ctx, "SELECT to_regclass('broker.broker_schema_migrations') IS NOT NULL").Scan(&exists)
 	if err != nil {
-		return fmt.Errorf("failed to read tables directory: %w", err)
+		return false, fmt.Errorf("failed to check migrations table: %w", err)
+	}
+	return exists, nil
+}
+
+// appliedVersions returns the set of migration versions already recorded.
+func (i *Installer) appliedVersions(ctx context.Context) (map[int64]bool, error) {
+	rows, err := i.db.Query(ctx, "SELECT version FROM broker.broker_schema_migrations")
+	if err != nil {
+		return nil, fmt.Errorf("failed to query applied migrations: %w", err)
+	}
+	defer rows.Close()
+
+	applied := make(map[int64]bool)
+	for rows.Next() {
+		var v int64
+		if err := rows.Scan(&v); err != nil {
+			return nil, fmt.Errorf("failed to scan migration version: %w", err)
+		}
+		applied[v] = true
+	}
+	return applied, rows.Err()
+}
+
+// PendingMigrations returns the names of embedded migrations that have not
+// yet been applied to the database, without applying them or creating the
+// migrations table -- safe to call with a least-privilege runtime role.
+func (i *Installer) PendingMigrations(ctx context.Context) ([]string, error) {
+	migrations, err := loadMigrations()
+	if err != nil {
+		return nil, err
 	}
 
-	// Filter and sort SQL files
+	exists, err := i.migrationsTableExists(ctx)
-	sqlFiles := filterAndSortSQLFiles(files)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		pending := make([]string, len(migrations))
+		for idx, m := range migrations {
+			pending[idx] = fmt.Sprintf("%04d_%s", m.version, m.name)
+		}
+		return pending, nil
+	}
 
-	for _, file := range sqlFiles {
+	applied, err := i.appliedVersions(ctx)
-		// Skip install script
+	if err != nil {
-		if file == "00_install.sql" {
+		return nil, err
+	}
+
+	var pending []string
+	for _, m := range migrations {
+		if !applied[m.version] {
+			pending = append(pending, fmt.Sprintf("%04d_%s", m.version, m.name))
+		}
+	}
+	return pending, nil
+}
+
+// ApplyMigrations applies every embedded migration that has not yet been
+// recorded in broker.broker_schema_migrations, each inside its own transaction.
+func (i *Installer) ApplyMigrations(ctx context.Context) error {
+	i.logger.Info("applying migrations")
+
+	if err := i.ensureMigrationsTable(ctx); err != nil {
+		return err
+	}
+
+	migrations, err := loadMigrations()
+	if err != nil {
+		return err
+	}
+
+	applied, err := i.appliedVersions(ctx)
+	if err != nil {
+		return err
+	}
+
+	appliedCount := 0
+	for _, m := range migrations {
+		if applied[m.version] {
 			continue
 		}
 
-		i.logger.Info("executing table script", "file", file)
+		content, err := migrationsFS.ReadFile(m.path)
-
-		content, err := sqlFS.ReadFile("sql/tables/" + file)
 		if err != nil {
-			return fmt.Errorf("failed to read file %s: %w", file, err)
+			return fmt.Errorf("failed to read migration %s: %w", m.path, err)
 		}
 
-		if err := i.executeSQL(ctx, string(content)); err != nil {
+		i.logger.Info("applying migration", "version", m.version, "name", m.name)
-			return fmt.Errorf("failed to execute %s: %w", file, err)
+
+		tx, err := i.db.Begin(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to begin transaction for migration %s: %w", m.name, err)
 		}
+
+		if err := execStatements(ctx, tx, string(content)); err != nil {
+			if rbErr := tx.Rollback(); rbErr != nil {
+				i.logger.Error("failed to rollback migration transaction", "version", m.version, "error", rbErr)
+			}
+			return fmt.Errorf("failed to apply migration %s: %w", m.name, err)
+		}
+
+		if _, err := tx.Exec(ctx,
+			"INSERT INTO broker.broker_schema_migrations (version, name) VALUES ($1, $2)",
+			m.version, m.name,
+		); err != nil {
+			if rbErr := tx.Rollback(); rbErr != nil {
+				i.logger.Error("failed to rollback migration transaction", "version", m.version, "error", rbErr)
+			}
+			return fmt.Errorf("failed to record migration %s: %w", m.name, err)
+		}
+
+		if err := tx.Commit(); err != nil {
+			return fmt.Errorf("failed to commit migration %s: %w", m.name, err)
+		}
+
+		appliedCount++
 	}
 
-	i.logger.Info("tables installed successfully")
+	if appliedCount == 0 {
+		i.logger.Info("no pending migrations")
+	} else {
+		i.logger.Info("migrations applied successfully", "count", appliedCount)
+	}
 	return nil
 }
 
-// installProcedures installs all stored procedures
+// RolePasswords holds the login passwords for the reference broker_admin,
-func (i *Installer) installProcedures(ctx context.Context) error {
+// broker_runtime, and broker_enqueue roles created by InstallRoles. All
-	i.logger.Info("installing procedures")
+// three are required -- there is no placeholder/default fallback, since
+// these roles carry real database privileges.
+type RolePasswords struct {
+	AdminPassword   string
+	RuntimePassword string
+	EnqueuePassword string
+}
 
-	files, err := sqlFS.ReadDir("sql/procedures")
+// InstallRoles applies the embedded role/grant scripts (sql/roles), which
+// create (or, if already present, rotate the password of) broker_admin,
+// broker_runtime, and broker_enqueue, then grant them the appropriate
+// schema/table/function privileges. The caller must connect as a superuser
+// or a role with CREATEROLE -- this is intentionally separate from the
+// migration-running connection.
+func (i *Installer) InstallRoles(ctx context.Context, passwords RolePasswords) error {
+	if passwords.AdminPassword == "" || passwords.RuntimePassword == "" || passwords.EnqueuePassword == "" {
+		return fmt.Errorf("all three role passwords (admin, runtime, enqueue) are required")
+	}
+
+	entries, err := rolesFS.ReadDir(rolesDir)
 	if err != nil {
-		return fmt.Errorf("failed to read procedures directory: %w", err)
+		return fmt.Errorf("failed to read roles directory: %w", err)
 	}
 
-	// Filter and sort SQL files
+	var names []string
-	sqlFiles := filterAndSortSQLFiles(files)
+	for _, e := range entries {
-
+		if !e.IsDir() {
-	for _, file := range sqlFiles {
+			names = append(names, e.Name())
-		// Skip install script
-		if file == "00_install.sql" {
-			continue
 		}
+	}
+	sort.Strings(names)
 
-		i.logger.Info("executing procedure script", "file", file)
+	replacer := strings.NewReplacer(
+		"__BROKER_ADMIN_PASSWORD__", pq.QuoteLiteral(passwords.AdminPassword),
+		"__BROKER_RUNTIME_PASSWORD__", pq.QuoteLiteral(passwords.RuntimePassword),
+		"__BROKER_ENQUEUE_PASSWORD__", pq.QuoteLiteral(passwords.EnqueuePassword),
+	)
 
-		content, err := sqlFS.ReadFile("sql/procedures/" + file)
+	for _, name := range names {
+		content, err := rolesFS.ReadFile(rolesDir + "/" + name)
 		if err != nil {
-			return fmt.Errorf("failed to read file %s: %w", file, err)
+			return fmt.Errorf("failed to read roles script %s: %w", name, err)
 		}
 
-		if err := i.executeSQL(ctx, string(content)); err != nil {
+		i.logger.Info("applying roles script", "name", name)
-			return fmt.Errorf("failed to execute %s: %w", file, err)
+
+		tx, err := i.db.Begin(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to begin transaction for roles script %s: %w", name, err)
+		}
+
+		// pq.QuoteLiteral already produces a safely quoted SQL string
+		// literal (doubling embedded quotes, or switching to E'...' escape
+		// syntax if the password contains a backslash), so this is a plain
+		// textual substitution, not string concatenation of untrusted input
+		// into SQL syntax.
+		rendered := replacer.Replace(string(content))
+
+		if err := execStatements(ctx, tx, rendered); err != nil {
+			if rbErr := tx.Rollback(); rbErr != nil {
+				i.logger.Error("failed to rollback roles script transaction", "name", name, "error", rbErr)
+			}
+			return fmt.Errorf("failed to apply roles script %s: %w", name, err)
+		}
+
+		if err := tx.Commit(); err != nil {
+			return fmt.Errorf("failed to commit roles script %s: %w", name, err)
 		}
 	}
 
-	i.logger.Info("procedures installed successfully")
+	i.logger.Info("roles installed successfully")
 	return nil
 }
 
-// executeSQL executes SQL statements
+// execStatements runs every statement in sql within tx.
-func (i *Installer) executeSQL(ctx context.Context, sql string) error {
+func execStatements(ctx context.Context, tx adapter.DBTransaction, sqlText string) error {
-	// Remove comments and split by statement
+	statements := splitSQLStatements(sqlText)
-	statements := splitSQLStatements(sql)
 
 	for _, stmt := range statements {
 		stmt = strings.TrimSpace(stmt)
-		if stmt == "" {
+		if stmt == "" || strings.HasPrefix(stmt, "\\") {
 			continue
 		}
-
+		if _, err := tx.Exec(ctx, stmt); err != nil {
-		// Skip psql-specific commands
-		if strings.HasPrefix(stmt, "\\") {
-			continue
-		}
-
-		if _, err := i.db.Exec(ctx, stmt); err != nil {
 			return fmt.Errorf("failed to execute statement: %w\nStatement: %s", err, stmt)
 		}
 	}
-
 	return nil
 }
 
-// filterAndSortSQLFiles filters and sorts SQL files
+// splitSQLStatements splits SQL into individual statements, keeping
-func filterAndSortSQLFiles(files []fs.DirEntry) []string {
+// $$-quoted function bodies intact.
-	var sqlFiles []string
+// splitSQLStatements splits a SQL script into individual statements on
-	for _, file := range files {
+// top-level semicolons, ignoring semicolons that appear inside single-quoted
-		if !file.IsDir() && strings.HasSuffix(file.Name(), ".sql") {
+// strings ('...', with ” as an escaped quote), double-quoted identifiers,
-			sqlFiles = append(sqlFiles, file.Name())
+// line comments (--), and dollar-quoted bodies ($$...$$ or $tag$...$tag$).
-		}
+func splitSQLStatements(sqlText string) []string {
-	}
-	sort.Strings(sqlFiles)
-	return sqlFiles
-}
-
-// splitSQLStatements splits SQL into individual statements
-func splitSQLStatements(sql string) []string {
-	// Simple split by semicolon
-	// This doesn't handle all edge cases (strings with semicolons, dollar-quoted strings, etc.)
-	// but works for our use case
-	statements := strings.Split(sql, ";")
-
 	var result []string
-	var buffer string
+	var buffer strings.Builder
 
-	for _, stmt := range statements {
+	runes := []rune(sqlText)
-		stmt = strings.TrimSpace(stmt)
+	n := len(runes)
-		if stmt == "" {
+	i := 0
+
+	for i < n {
+		c := runes[i]
+
+		switch {
+		case c == '-' && i+1 < n && runes[i+1] == '-':
+			// Line comment: copy through end of line.
+			for i < n && runes[i] != '\n' {
+				buffer.WriteRune(runes[i])
+				i++
+			}
 			continue
-		}
 
-		buffer += stmt + ";"
+		case c == '\'':
+			buffer.WriteRune(c)
+			i++
+			for i < n {
+				buffer.WriteRune(runes[i])
+				if runes[i] == '\'' {
+					if i+1 < n && runes[i+1] == '\'' {
+						buffer.WriteRune(runes[i+1])
+						i += 2
+						continue
+					}
+					i++
+					break
+				}
+				i++
+			}
+			continue
 
-		// Check if we're inside a function definition ($$)
+		case c == '"':
-		dollarCount := strings.Count(buffer, "$$")
+			buffer.WriteRune(c)
-		if dollarCount%2 == 0 {
+			i++
-			// Even number of $$ means we're outside function definitions
+			for i < n {
-			result = append(result, buffer)
+				buffer.WriteRune(runes[i])
-			buffer = ""
+				if runes[i] == '"' {
-		} else {
+					i++
-			// Odd number means we're inside a function, keep accumulating
+					break
-			buffer += " "
+				}
+				i++
+			}
+			continue
+
+		case c == '$':
+			if tag, ok := matchDollarTag(runes, i); ok {
+				closer := tag
+				buffer.WriteString(closer)
+				i += len(closer)
+				end := indexOfRunes(runes, i, closer)
+				if end == -1 {
+					buffer.WriteString(string(runes[i:]))
+					i = n
+				} else {
+					buffer.WriteString(string(runes[i:end]))
+					buffer.WriteString(closer)
+					i = end + len(closer)
+				}
+				continue
+			}
+			buffer.WriteRune(c)
+			i++
+
+		case c == ';':
+			stmt := strings.TrimSpace(buffer.String())
+			if stmt != "" {
+				result = append(result, stmt+";")
+			}
+			buffer.Reset()
+			i++
+
+		default:
+			buffer.WriteRune(c)
+			i++
 		}
 	}
 
-	// Add any remaining buffered content
+	if stmt := strings.TrimSpace(buffer.String()); stmt != "" {
-	if buffer != "" {
+		result = append(result, stmt)
-		result = append(result, buffer)
 	}
 
 	return result
 }
 
-// VerifyInstallation checks if the schema is properly installed
+// matchDollarTag checks whether runes[pos:] begins a dollar-quote tag
+// ($$ or $tag$) and returns that tag if so.
+func matchDollarTag(runes []rune, pos int) (string, bool) {
+	if runes[pos] != '$' {
+		return "", false
+	}
+	j := pos + 1
+	for j < len(runes) && (runes[j] == '_' || isAlnum(runes[j])) {
+		j++
+	}
+	if j < len(runes) && runes[j] == '$' {
+		return string(runes[pos : j+1]), true
+	}
+	return "", false
+}
+
+func isAlnum(r rune) bool {
+	return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')
+}
+
+// indexOfRunes returns the index of the first occurrence of sub in
+// runes[from:], or -1 if not found.
+func indexOfRunes(runes []rune, from int, sub string) int {
+	subRunes := []rune(sub)
+	for i := from; i+len(subRunes) <= len(runes); i++ {
+		match := true
+		for j, r := range subRunes {
+			if runes[i+j] != r {
+				match = false
+				break
+			}
+		}
+		if match {
+			return i
+		}
+	}
+	return -1
+}
+
+// VerifyInstallation checks that every embedded migration has been applied.
 func (i *Installer) VerifyInstallation(ctx context.Context) error {
 	i.logger.Info("verifying installation")
 
-	tables := []string{"broker_queueinstance", "broker_jobs", "broker_schedule"}
+	pending, err := i.PendingMigrations(ctx)
-	procedures := []string{
+	if err != nil {
-		"broker_get",
+		return fmt.Errorf("failed to check pending migrations: %w", err)
-		"broker_run",
-		"broker_set",
-		"broker_add_job",
-		"broker_register_instance",
-		"broker_ping_instance",
-		"broker_shutdown_instance",
 	}
 
-	// Check tables
+	if len(pending) > 0 {
-	for _, table := range tables {
+		return fmt.Errorf("schema is behind: %d migration(s) not applied: %s", len(pending), strings.Join(pending, ", "))
-		var exists bool
-		err := i.db.QueryRow(ctx,
-			"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $1)",
-			table,
-		).Scan(&exists)
-
-		if err != nil {
-			return fmt.Errorf("failed to check table %s: %w", table, err)
-		}
-
-		if !exists {
-			return fmt.Errorf("table %s does not exist", table)
-		}
-
-		i.logger.Info("table verified", "table", table)
-	}
-
-	// Check procedures
-	for _, proc := range procedures {
-		var exists bool
-		err := i.db.QueryRow(ctx,
-			"SELECT EXISTS (SELECT FROM pg_proc WHERE proname = $1)",
-			proc,
-		).Scan(&exists)
-
-		if err != nil {
-			return fmt.Errorf("failed to check procedure %s: %w", proc, err)
-		}
-
-		if !exists {
-			return fmt.Errorf("procedure %s does not exist", proc)
-		}
-
-		i.logger.Info("procedure verified", "procedure", proc)
 	}
 
 	i.logger.Info("installation verified successfully")

pkg/broker/install/sql/migrations/0001_schema.sql

@@ -0,0 +1,6 @@
+-- Dedicated schema for all broker objects.
+CREATE SCHEMA IF NOT EXISTS broker;
+REVOKE ALL ON SCHEMA broker FROM PUBLIC;
+
+-- gen_random_uuid() for lease tokens.
+CREATE EXTENSION IF NOT EXISTS pgcrypto;

pkg/broker/install/sql/migrations/0002_broker_queueinstance.sql

@@ -0,0 +1,25 @@
+-- broker.broker_queueinstance
+-- Tracks active and historical broker queue instances.
+
+CREATE TABLE IF NOT EXISTS broker.broker_queueinstance (
+    id_broker_queueinstance BIGSERIAL PRIMARY KEY,
+    name VARCHAR(255) NOT NULL,
+    hostname VARCHAR(255) NOT NULL,
+    pid INTEGER NOT NULL,
+    version VARCHAR(50) NOT NULL,
+    status VARCHAR(20) NOT NULL DEFAULT 'active',
+    started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    last_ping_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    shutdown_at TIMESTAMP WITH TIME ZONE,
+    queue_count INTEGER NOT NULL DEFAULT 0,
+    jobs_handled BIGINT NOT NULL DEFAULT 0,
+
+    CONSTRAINT broker_queueinstance_status_check CHECK (status IN ('active', 'inactive', 'shutdown'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_broker_queueinstance_status ON broker.broker_queueinstance(status);
+CREATE INDEX IF NOT EXISTS idx_broker_queueinstance_hostname ON broker.broker_queueinstance(hostname);
+CREATE INDEX IF NOT EXISTS idx_broker_queueinstance_last_ping ON broker.broker_queueinstance(last_ping_at);
+
+COMMENT ON TABLE broker.broker_queueinstance IS 'Tracks broker queue instances (active and historical). Single-active-instance-per-name is enforced via a pg_try_advisory_lock in broker_register_instance, not by this status column, which is observational only.';
+COMMENT ON COLUMN broker.broker_queueinstance.status IS 'Observational status: active, inactive, or shutdown. Ownership is enforced via advisory lock, not by reading this column.';

pkg/broker/install/sql/migrations/0003_broker_schedule.sql

@@ -0,0 +1,41 @@
+-- broker.broker_schedule
+-- Stores scheduled jobs (cron-like functionality).
+
+CREATE TABLE IF NOT EXISTS broker.broker_schedule (
+    id_broker_schedule BIGSERIAL PRIMARY KEY,
+    name VARCHAR(255) NOT NULL UNIQUE,
+    cron_expr VARCHAR(100) NOT NULL,
+    enabled BOOLEAN NOT NULL DEFAULT true,
+    job_name VARCHAR(255) NOT NULL,
+    job_priority INTEGER NOT NULL DEFAULT 0,
+    job_queue INTEGER NOT NULL DEFAULT 1,
+    job_language VARCHAR(50) NOT NULL DEFAULT 'sql',
+    execute_str TEXT NOT NULL,
+    run_as VARCHAR(100),
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    last_run_at TIMESTAMP WITH TIME ZONE,
+    next_run_at TIMESTAMP WITH TIME ZONE,
+
+    CONSTRAINT broker_schedule_job_queue_check CHECK (job_queue > 0)
+);
+
+CREATE INDEX IF NOT EXISTS idx_broker_schedule_enabled ON broker.broker_schedule(enabled);
+CREATE INDEX IF NOT EXISTS idx_broker_schedule_next_run ON broker.broker_schedule(next_run_at) WHERE enabled = true;
+CREATE INDEX IF NOT EXISTS idx_broker_schedule_name ON broker.broker_schedule(name);
+
+COMMENT ON TABLE broker.broker_schedule IS 'Scheduled jobs (cron-like functionality)';
+
+CREATE OR REPLACE FUNCTION broker.tf_broker_schedule_update_timestamp()
+RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated_at = NOW();
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS t_broker_schedule_updated_at ON broker.broker_schedule;
+CREATE TRIGGER t_broker_schedule_updated_at
+    BEFORE UPDATE ON broker.broker_schedule
+    FOR EACH ROW
+    EXECUTE FUNCTION broker.tf_broker_schedule_update_timestamp();

pkg/broker/install/sql/migrations/0004_broker_jobs.sql

@@ -0,0 +1,93 @@
+-- broker.broker_jobs
+-- Job queue for broker execution.
+-- tenant_id / RLS: rows are only visible/writable when tenant_id matches
+-- current_setting('broker.tenant_id', true) for the current transaction.
+-- Callers must invoke broker.broker_set_tenant(...) before enqueue/claim;
+-- if they don't, tenant_id defaults to 'default' and current_setting
+-- also defaults to NULL -> broker_add_job coalesces to 'default' so
+-- single-tenant use keeps working unmodified.
+
+CREATE TABLE IF NOT EXISTS broker.broker_jobs (
+    id_broker_jobs BIGSERIAL PRIMARY KEY,
+    job_name VARCHAR(255) NOT NULL,
+    job_priority INTEGER NOT NULL DEFAULT 0,
+    job_queue INTEGER NOT NULL DEFAULT 1,
+    job_language VARCHAR(50) NOT NULL DEFAULT 'sql',
+    execute_str TEXT NOT NULL,
+    execute_result TEXT,
+    error_msg TEXT,
+    complete_status INTEGER NOT NULL DEFAULT 0,
+    run_as VARCHAR(100),
+    rid_broker_schedule BIGINT,
+    rid_broker_queueinstance BIGINT,
+    tenant_id TEXT NOT NULL DEFAULT 'default',
+
+    -- Lease / retry / idempotency
+    attempt_count INTEGER NOT NULL DEFAULT 0,
+    max_attempts INTEGER NOT NULL DEFAULT 1,
+    available_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    leased_at TIMESTAMP WITH TIME ZONE,
+    lease_expires_at TIMESTAMP WITH TIME ZONE,
+    lease_token UUID,
+    idempotency_key TEXT,
+
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    started_at TIMESTAMP WITH TIME ZONE,
+    completed_at TIMESTAMP WITH TIME ZONE,
+
+    CONSTRAINT broker_jobs_complete_status_check CHECK (complete_status IN (0, 1, 2, 3, 4)),
+    CONSTRAINT broker_jobs_job_queue_check CHECK (job_queue > 0),
+    CONSTRAINT fk_schedule FOREIGN KEY (rid_broker_schedule) REFERENCES broker.broker_schedule(id_broker_schedule) ON DELETE SET NULL,
+    CONSTRAINT fk_instance FOREIGN KEY (rid_broker_queueinstance) REFERENCES broker.broker_queueinstance(id_broker_queueinstance) ON DELETE SET NULL
+);
+
+-- General-purpose indexes
+CREATE INDEX IF NOT EXISTS idx_broker_jobs_status ON broker.broker_jobs(complete_status);
+CREATE INDEX IF NOT EXISTS idx_broker_jobs_schedule ON broker.broker_jobs(rid_broker_schedule);
+CREATE INDEX IF NOT EXISTS idx_broker_jobs_instance ON broker.broker_jobs(rid_broker_queueinstance);
+CREATE INDEX IF NOT EXISTS idx_broker_jobs_created ON broker.broker_jobs(created_at);
+CREATE INDEX IF NOT EXISTS idx_broker_jobs_name ON broker.broker_jobs(job_name, complete_status);
+CREATE INDEX IF NOT EXISTS idx_broker_jobs_tenant ON broker.broker_jobs(tenant_id);
+
+-- Claim index: exactly what broker_get's WHERE/ORDER BY needs, partial on pending rows only.
+CREATE INDEX IF NOT EXISTS idx_broker_jobs_claim
+    ON broker.broker_jobs (job_queue, job_priority DESC, created_at, id_broker_jobs)
+    WHERE complete_status = 0;
+
+-- Idempotency: at most one pending/any job per (queue, key) when a key is supplied.
+CREATE UNIQUE INDEX IF NOT EXISTS idx_broker_jobs_idempotency
+    ON broker.broker_jobs (job_queue, idempotency_key)
+    WHERE idempotency_key IS NOT NULL;
+
+COMMENT ON TABLE broker.broker_jobs IS 'Job queue for broker execution';
+COMMENT ON COLUMN broker.broker_jobs.complete_status IS '0=pending, 1=running, 2=completed, 3=failed (terminal or dead-lettered once attempt_count>=max_attempts), 4=cancelled';
+COMMENT ON COLUMN broker.broker_jobs.tenant_id IS 'RLS tenant scaffold; defaults to ''default'' for single-tenant use';
+COMMENT ON COLUMN broker.broker_jobs.attempt_count IS 'Number of times this job has been claimed/executed';
+COMMENT ON COLUMN broker.broker_jobs.max_attempts IS 'Job is dead-lettered (failed) once attempt_count reaches this value';
+COMMENT ON COLUMN broker.broker_jobs.available_at IS 'Job is not claimable until now() >= available_at (used for retry backoff)';
+COMMENT ON COLUMN broker.broker_jobs.lease_token IS 'Token handed out by broker_get; broker_run requires a matching token to execute, so an expired/reclaimed lease cannot be double-processed';
+COMMENT ON COLUMN broker.broker_jobs.idempotency_key IS 'Optional caller-supplied key; unique per (job_queue, idempotency_key)';
+
+CREATE OR REPLACE FUNCTION broker.tf_broker_jobs_update_timestamp()
+RETURNS TRIGGER AS $$
+BEGIN
+    NEW.updated_at = NOW();
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS t_broker_jobs_updated_at ON broker.broker_jobs;
+CREATE TRIGGER t_broker_jobs_updated_at
+    BEFORE UPDATE ON broker.broker_jobs
+    FOR EACH ROW
+    EXECUTE FUNCTION broker.tf_broker_jobs_update_timestamp();
+
+-- Row Level Security: tenant isolation
+ALTER TABLE broker.broker_jobs ENABLE ROW LEVEL SECURITY;
+ALTER TABLE broker.broker_jobs FORCE ROW LEVEL SECURITY;
+
+DROP POLICY IF EXISTS broker_jobs_tenant_isolation ON broker.broker_jobs;
+CREATE POLICY broker_jobs_tenant_isolation ON broker.broker_jobs
+    USING (tenant_id = COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default'))
+    WITH CHECK (tenant_id = COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default'));

pkg/broker/install/sql/migrations/0005_broker_job_dependency.sql

@@ -0,0 +1,27 @@
+-- broker.broker_job_dependency
+-- Replaces the old broker_jobs.depends_on text[] column: job_id is only
+-- claimable once every row it depends on has complete_status = 2 (completed).
+
+CREATE TABLE IF NOT EXISTS broker.broker_job_dependency (
+    job_id BIGINT NOT NULL REFERENCES broker.broker_jobs(id_broker_jobs) ON DELETE CASCADE,
+    depends_on_job_id BIGINT NOT NULL REFERENCES broker.broker_jobs(id_broker_jobs) ON DELETE CASCADE,
+    tenant_id TEXT NOT NULL DEFAULT 'default',
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    PRIMARY KEY (job_id, depends_on_job_id),
+    CHECK (job_id <> depends_on_job_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_broker_job_dependency_reverse
+    ON broker.broker_job_dependency (depends_on_job_id, job_id);
+CREATE INDEX IF NOT EXISTS idx_broker_job_dependency_tenant
+    ON broker.broker_job_dependency (tenant_id);
+
+COMMENT ON TABLE broker.broker_job_dependency IS 'job_id is not claimable until every depends_on_job_id row has complete_status = 2 (completed)';
+
+ALTER TABLE broker.broker_job_dependency ENABLE ROW LEVEL SECURITY;
+ALTER TABLE broker.broker_job_dependency FORCE ROW LEVEL SECURITY;
+
+DROP POLICY IF EXISTS broker_job_dependency_tenant_isolation ON broker.broker_job_dependency;
+CREATE POLICY broker_job_dependency_tenant_isolation ON broker.broker_job_dependency
+    USING (tenant_id = COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default'))
+    WITH CHECK (tenant_id = COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default'));

pkg/broker/install/sql/migrations/0006_broker_set_tenant.sql

@@ -0,0 +1,13 @@
+-- broker.broker_set_tenant
+-- Sets the RLS tenant context for the current transaction (SET LOCAL semantics
+-- via set_config(..., true)). Callers must invoke this before enqueue/claim
+-- if they are not using the 'default' tenant.
+
+CREATE OR REPLACE FUNCTION broker.broker_set_tenant(p_tenant_id TEXT)
+RETURNS VOID
+LANGUAGE SQL
+AS $$
+    SELECT set_config('broker.tenant_id', p_tenant_id, true);
+$$;
+
+COMMENT ON FUNCTION broker.broker_set_tenant IS 'Sets broker.tenant_id for the current transaction only (SET LOCAL semantics)';

pkg/broker/install/sql/migrations/0007_broker_get.sql

@@ -0,0 +1,83 @@
+-- broker.broker_get
+-- Claims the next eligible job from a queue: pending, available (backoff
+-- elapsed), no incomplete dependency, and visible under the caller's RLS
+-- tenant. Grants a lease (lease_token) that broker_run must present back.
+-- Returns: p_retval (0=success, >0=infra error), p_errmsg, p_job_id, p_lease_token.
+
+CREATE OR REPLACE FUNCTION broker.broker_get(
+    p_queue_number INTEGER,
+    p_instance_id BIGINT DEFAULT NULL,
+    p_lease_seconds INTEGER DEFAULT 60,
+    OUT p_retval INTEGER,
+    OUT p_errmsg TEXT,
+    OUT p_job_id BIGINT,
+    OUT p_lease_token UUID
+)
+RETURNS RECORD
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    v_job_id BIGINT;
+    v_lease_token UUID;
+BEGIN
+    p_retval := 0;
+    p_errmsg := '';
+    p_job_id := NULL;
+    p_lease_token := NULL;
+
+    IF p_queue_number IS NULL OR p_queue_number <= 0 THEN
+        p_retval := 1;
+        p_errmsg := 'Invalid queue number';
+        RETURN;
+    END IF;
+
+    IF p_lease_seconds IS NULL OR p_lease_seconds <= 0 THEN
+        p_lease_seconds := 60;
+    END IF;
+
+    SELECT candidate.id_broker_jobs
+    INTO v_job_id
+    FROM broker.broker_jobs candidate
+    WHERE candidate.job_queue = p_queue_number
+      AND candidate.complete_status = 0
+      AND candidate.available_at <= NOW()
+      AND NOT EXISTS (
+        SELECT 1
+        FROM broker.broker_job_dependency d
+        JOIN broker.broker_jobs dep ON dep.id_broker_jobs = d.depends_on_job_id
+        WHERE d.job_id = candidate.id_broker_jobs
+          AND dep.complete_status <> 2
+      )
+    ORDER BY candidate.job_priority DESC, candidate.created_at ASC, candidate.id_broker_jobs ASC
+    LIMIT 1
+    FOR UPDATE SKIP LOCKED;
+
+    IF NOT FOUND THEN
+        RETURN;
+    END IF;
+
+    v_lease_token := gen_random_uuid();
+
+    UPDATE broker.broker_jobs
+    SET complete_status = 1,  -- running
+        started_at = NOW(),
+        rid_broker_queueinstance = p_instance_id,
+        attempt_count = attempt_count + 1,
+        lease_token = v_lease_token,
+        leased_at = NOW(),
+        lease_expires_at = NOW() + make_interval(secs => p_lease_seconds),
+        updated_at = NOW()
+    WHERE id_broker_jobs = v_job_id;
+
+    p_job_id := v_job_id;
+    p_lease_token := v_lease_token;
+
+EXCEPTION
+    WHEN OTHERS THEN
+        p_retval := 2;
+        p_errmsg := SQLERRM;
+        RAISE WARNING 'broker_get error: %', SQLERRM;
+END;
+$$;
+
+COMMENT ON FUNCTION broker.broker_get IS 'Claims the next eligible job from a queue and grants a lease';

pkg/broker/install/sql/migrations/0008_broker_run.sql

@@ -0,0 +1,135 @@
+-- broker.broker_run
+-- Executes a job by its ID, presenting the lease token it was claimed with.
+--
+-- p_retval is reserved for infra failures (bad job id, job not found, wrong
+-- state, lease mismatch/expired, DB error). An executed-and-caught job
+-- failure is a *successful* invocation: p_retval stays 0 and the outcome is
+-- reported via p_job_status (0=requeued for retry, 2=completed, 3=dead-lettered)
+-- so the caller commits the terminal/retry state instead of rolling it back.
+--
+-- On failure, if attempt_count < max_attempts the job is reset to pending
+-- with exponential backoff (base 5s, capped at 300s); otherwise it is
+-- dead-lettered as complete_status = 3.
+
+CREATE OR REPLACE FUNCTION broker.broker_run(
+    p_job_id BIGINT,
+    p_lease_token UUID,
+    OUT p_retval INTEGER,
+    OUT p_errmsg TEXT,
+    OUT p_job_status INTEGER
+)
+RETURNS RECORD
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    v_job_record RECORD;
+    v_execute_result TEXT;
+    v_error_occurred BOOLEAN := false;
+    v_backoff_base CONSTANT INTEGER := 5;
+    v_backoff_cap CONSTANT INTEGER := 300;
+    v_backoff_secs INTEGER;
+BEGIN
+    p_retval := 0;
+    p_errmsg := '';
+    p_job_status := NULL;
+    v_execute_result := '';
+
+    IF p_job_id IS NULL OR p_job_id <= 0 THEN
+        p_retval := 1;
+        p_errmsg := 'Invalid job ID';
+        RETURN;
+    END IF;
+
+    SELECT id_broker_jobs, execute_str, job_language, complete_status, attempt_count, max_attempts, lease_token
+    INTO v_job_record
+    FROM broker.broker_jobs
+    WHERE id_broker_jobs = p_job_id
+    FOR UPDATE;
+
+    IF NOT FOUND THEN
+        p_retval := 2;
+        p_errmsg := 'Job not found';
+        RETURN;
+    END IF;
+
+    IF v_job_record.complete_status != 1 THEN
+        p_retval := 3;
+        p_errmsg := format('Job is not in running state (status: %s)', v_job_record.complete_status);
+        RETURN;
+    END IF;
+
+    IF v_job_record.lease_token IS DISTINCT FROM p_lease_token THEN
+        p_retval := 4;
+        p_errmsg := 'Lease token mismatch or expired; job was reclaimed by another worker';
+        RETURN;
+    END IF;
+
+    -- Execute the job
+    BEGIN
+        IF v_job_record.job_language IN ('sql', 'plpgsql') THEN
+            EXECUTE v_job_record.execute_str;
+            v_execute_result := 'Success';
+        ELSE
+            v_error_occurred := true;
+            v_execute_result := format('Unsupported job language: %s', v_job_record.job_language);
+        END IF;
+    EXCEPTION
+        WHEN OTHERS THEN
+            v_error_occurred := true;
+            v_execute_result := format('Error: %s', SQLERRM);
+    END;
+
+    IF v_error_occurred THEN
+        IF v_job_record.attempt_count < v_job_record.max_attempts THEN
+            v_backoff_secs := LEAST(POWER(2, v_job_record.attempt_count)::INTEGER * v_backoff_base, v_backoff_cap);
+
+            UPDATE broker.broker_jobs
+            SET complete_status = 0,  -- pending, retry
+                available_at = NOW() + make_interval(secs => v_backoff_secs),
+                error_msg = v_execute_result,
+                execute_result = v_execute_result,
+                lease_token = NULL,
+                leased_at = NULL,
+                lease_expires_at = NULL,
+                updated_at = NOW()
+            WHERE id_broker_jobs = p_job_id;
+
+            p_job_status := 0;
+        ELSE
+            UPDATE broker.broker_jobs
+            SET complete_status = 3,  -- failed (dead-letter, attempts exhausted)
+                error_msg = v_execute_result,
+                execute_result = v_execute_result,
+                lease_token = NULL,
+                leased_at = NULL,
+                lease_expires_at = NULL,
+                completed_at = NOW(),
+                updated_at = NOW()
+            WHERE id_broker_jobs = p_job_id;
+
+            p_job_status := 3;
+        END IF;
+    ELSE
+        UPDATE broker.broker_jobs
+        SET complete_status = 2,  -- completed
+            execute_result = v_execute_result,
+            error_msg = NULL,
+            lease_token = NULL,
+            leased_at = NULL,
+            lease_expires_at = NULL,
+            completed_at = NOW(),
+            updated_at = NOW()
+        WHERE id_broker_jobs = p_job_id;
+
+        p_job_status := 2;
+    END IF;
+
+EXCEPTION
+    WHEN OTHERS THEN
+        p_retval := 6;
+        p_errmsg := SQLERRM;
+        RAISE WARNING 'broker_run error: %', SQLERRM;
+END;
+$$;
+
+COMMENT ON FUNCTION broker.broker_run IS 'Executes a leased job; reports outcome via p_job_status without forcing a rollback of the terminal/retry state';

pkg/broker/install/sql/migrations/0009_broker_set.sql

@@ -0,0 +1,65 @@
+-- broker.broker_set
+-- Minimal whitelist of session options. The previous SET SESSION AUTHORIZATION
+-- and search_path branches were removed: they let a caller assume an arbitrary
+-- Postgres role or schema search order from inside a plpgsql function with no
+-- identity model behind it, which is unsafe and was unused.
+
+CREATE OR REPLACE FUNCTION broker.broker_set(
+    p_option_name TEXT,
+    p_option_value TEXT,
+    OUT p_retval INTEGER,
+    OUT p_errmsg TEXT
+)
+RETURNS RECORD
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    v_sql TEXT;
+BEGIN
+    p_retval := 0;
+    p_errmsg := '';
+
+    IF p_option_name IS NULL OR p_option_name = '' THEN
+        p_retval := 1;
+        p_errmsg := 'Option name is required';
+        RETURN;
+    END IF;
+
+    CASE LOWER(p_option_name)
+        WHEN 'application_name' THEN
+            BEGIN
+                v_sql := format('SET LOCAL application_name TO %L', p_option_value);
+                EXECUTE v_sql;
+            EXCEPTION
+                WHEN OTHERS THEN
+                    p_retval := 3;
+                    p_errmsg := format('Failed to set application_name: %s', SQLERRM);
+                    RETURN;
+            END;
+
+        WHEN 'timezone' THEN
+            BEGIN
+                v_sql := format('SET LOCAL timezone TO %L', p_option_value);
+                EXECUTE v_sql;
+            EXCEPTION
+                WHEN OTHERS THEN
+                    p_retval := 5;
+                    p_errmsg := format('Failed to set timezone: %s', SQLERRM);
+                    RETURN;
+            END;
+
+        ELSE
+            p_retval := 10;
+            p_errmsg := format('Unknown option: %s', p_option_name);
+            RETURN;
+    END CASE;
+
+EXCEPTION
+    WHEN OTHERS THEN
+        p_retval := 99;
+        p_errmsg := SQLERRM;
+        RAISE WARNING 'broker_set error: %', SQLERRM;
+END;
+$$;
+
+COMMENT ON FUNCTION broker.broker_set IS 'Sets a whitelisted session-local option (application_name, timezone)';

pkg/broker/install/sql/migrations/0010_broker_register_instance.sql

@@ -0,0 +1,64 @@
+-- broker.broker_register_instance
+-- Registers a broker instance, using a session-scoped advisory lock keyed by
+-- name to guarantee only one active instance per name -- no race window, no
+-- "check COUNT(*) then insert" gap. The caller MUST run this on a pinned
+-- connection it keeps open for the process lifetime (Go: db.Conn(ctx)) and
+-- release the lock (pg_advisory_unlock) itself on shutdown, since the lock
+-- lives with the backend session, not with the row.
+
+CREATE OR REPLACE FUNCTION broker.broker_register_instance(
+    p_name TEXT,
+    p_hostname TEXT,
+    p_pid INTEGER,
+    p_version TEXT,
+    p_queue_count INTEGER,
+    OUT p_retval INTEGER,
+    OUT p_errmsg TEXT,
+    OUT p_instance_id BIGINT
+)
+RETURNS RECORD
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    v_lock_key BIGINT;
+BEGIN
+    p_retval := 0;
+    p_errmsg := '';
+    p_instance_id := NULL;
+
+    IF p_name IS NULL OR p_name = '' THEN
+        p_retval := 1;
+        p_errmsg := 'Instance name is required';
+        RETURN;
+    END IF;
+
+    IF p_hostname IS NULL OR p_hostname = '' THEN
+        p_retval := 2;
+        p_errmsg := 'Hostname is required';
+        RETURN;
+    END IF;
+
+    v_lock_key := hashtextextended('broker:' || p_name, 0);
+
+    IF NOT pg_try_advisory_lock(v_lock_key) THEN
+        p_retval := 3;
+        p_errmsg := 'Another broker instance is already active for this name (advisory lock held). Only one broker instance per name is allowed.';
+        RETURN;
+    END IF;
+
+    INSERT INTO broker.broker_queueinstance (
+        name, hostname, pid, version, status, queue_count, started_at, last_ping_at
+    ) VALUES (
+        p_name, p_hostname, p_pid, p_version, 'active', p_queue_count, NOW(), NOW()
+    )
+    RETURNING id_broker_queueinstance INTO p_instance_id;
+
+EXCEPTION
+    WHEN OTHERS THEN
+        p_retval := 99;
+        p_errmsg := SQLERRM;
+        RAISE WARNING 'broker_register_instance error: %', SQLERRM;
+END;
+$$;
+
+COMMENT ON FUNCTION broker.broker_register_instance IS 'Registers a broker instance; caller must hold the connection open for the process lifetime (advisory lock is session-scoped)';

pkg/broker/install/sql/migrations/0011_broker_add_job.sql

@@ -0,0 +1,137 @@
+-- broker.broker_add_job
+-- Adds a new job (optionally with dependencies and an idempotency key) and
+-- sends a wake-only NOTIFY -- the payload carries only the queue number
+-- (job id kept solely for logging); workers re-claim via broker_get rather
+-- than executing the notified row directly, so a notification can never
+-- hand a job to a worker before it's actually claimable.
+
+CREATE OR REPLACE FUNCTION broker.broker_add_job(
+    p_job_name TEXT,
+    p_execute_str TEXT,
+    p_job_queue INTEGER DEFAULT 1,
+    p_job_priority INTEGER DEFAULT 0,
+    p_job_language TEXT DEFAULT 'sql',
+    p_run_as TEXT DEFAULT NULL,
+    p_schedule_id BIGINT DEFAULT NULL,
+    p_depends_on_job_ids BIGINT[] DEFAULT NULL,
+    p_idempotency_key TEXT DEFAULT NULL,
+    p_max_attempts INTEGER DEFAULT 1,
+    OUT p_retval INTEGER,
+    OUT p_errmsg TEXT,
+    OUT p_job_id BIGINT
+)
+RETURNS RECORD
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    v_notification_payload JSON;
+    v_tenant_id TEXT;
+    v_dep_id BIGINT;
+    v_cycle_exists BOOLEAN;
+BEGIN
+    p_retval := 0;
+    p_errmsg := '';
+    p_job_id := NULL;
+
+    IF p_job_name IS NULL OR p_job_name = '' THEN
+        p_retval := 1;
+        p_errmsg := 'Job name is required';
+        RETURN;
+    END IF;
+
+    IF p_execute_str IS NULL OR p_execute_str = '' THEN
+        p_retval := 2;
+        p_errmsg := 'Execute string is required';
+        RETURN;
+    END IF;
+
+    IF p_job_queue IS NULL OR p_job_queue <= 0 THEN
+        p_retval := 3;
+        p_errmsg := 'Invalid job queue number';
+        RETURN;
+    END IF;
+
+    IF p_max_attempts IS NULL OR p_max_attempts <= 0 THEN
+        p_max_attempts := 1;
+    END IF;
+
+    -- Falls back to 'default' when the caller never called broker_set_tenant,
+    -- so single-tenant use (and the RLS WITH CHECK on insert) keeps working.
+    v_tenant_id := COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default');
+
+    IF p_idempotency_key IS NOT NULL THEN
+        SELECT id_broker_jobs INTO p_job_id
+        FROM broker.broker_jobs
+        WHERE job_queue = p_job_queue
+          AND idempotency_key = p_idempotency_key
+          AND tenant_id = v_tenant_id;
+
+        IF FOUND THEN
+            p_errmsg := 'Job with this idempotency key already exists; returning existing job id';
+            RETURN;
+        END IF;
+    END IF;
+
+    INSERT INTO broker.broker_jobs (
+        job_name, job_priority, job_queue, job_language, execute_str, run_as,
+        rid_broker_schedule, tenant_id, max_attempts, idempotency_key, complete_status
+    ) VALUES (
+        p_job_name, p_job_priority, p_job_queue, p_job_language, p_execute_str, p_run_as,
+        p_schedule_id, v_tenant_id, p_max_attempts, p_idempotency_key, 0
+    )
+    RETURNING id_broker_jobs INTO p_job_id;
+
+    IF p_depends_on_job_ids IS NOT NULL THEN
+        FOREACH v_dep_id IN ARRAY p_depends_on_job_ids LOOP
+            IF v_dep_id IS NULL THEN
+                CONTINUE;
+            END IF;
+
+            IF v_dep_id = p_job_id THEN
+                p_retval := 20;
+                p_errmsg := 'Invalid dependency: a job cannot depend on itself';
+                RETURN;
+            END IF;
+
+            SELECT EXISTS (
+                SELECT 1 FROM broker.broker_job_dependency
+                WHERE job_id = v_dep_id AND depends_on_job_id = p_job_id
+            ) INTO v_cycle_exists;
+
+            IF v_cycle_exists THEN
+                p_retval := 21;
+                p_errmsg := format('Invalid dependency: job %s already depends on %s (would create a cycle)', v_dep_id, p_job_id);
+                RETURN;
+            END IF;
+
+            INSERT INTO broker.broker_job_dependency (job_id, depends_on_job_id, tenant_id)
+            VALUES (p_job_id, v_dep_id, v_tenant_id)
+            ON CONFLICT (job_id, depends_on_job_id) DO NOTHING;
+        END LOOP;
+    END IF;
+
+    v_notification_payload := json_build_object(
+        'queue', p_job_queue,
+        'job_id', p_job_id
+    );
+
+    PERFORM pg_notify('broker.event', v_notification_payload::text);
+
+EXCEPTION
+    WHEN unique_violation THEN
+        -- Concurrent insert raced us to the same idempotency key.
+        p_retval := 0;
+        p_errmsg := 'Job with this idempotency key already exists; returning existing job id';
+        SELECT id_broker_jobs INTO p_job_id
+        FROM broker.broker_jobs
+        WHERE job_queue = p_job_queue
+          AND idempotency_key = p_idempotency_key
+          AND tenant_id = v_tenant_id;
+    WHEN OTHERS THEN
+        p_retval := 99;
+        p_errmsg := SQLERRM;
+        RAISE WARNING 'broker_add_job error: %', SQLERRM;
+END;
+$$;
+
+COMMENT ON FUNCTION broker.broker_add_job IS 'Adds a job (with optional dependencies/idempotency key) and sends a wake-only NOTIFY';

pkg/broker/install/sql/migrations/0012_broker_ping_shutdown_instance.sql

@@ -1,8 +1,6 @@
--- broker_ping_instance function
+-- broker.broker_ping_instance / broker.broker_shutdown_instance
--- Updates the last_ping_at timestamp for a broker instance
--- Returns: p_retval (0=success, >0=error), p_errmsg (error message)
 
-CREATE OR REPLACE FUNCTION broker_ping_instance(
+CREATE OR REPLACE FUNCTION broker.broker_ping_instance(
     p_instance_id BIGINT,
     p_jobs_handled BIGINT DEFAULT NULL,
     OUT p_retval INTEGER,
@@ -15,26 +13,22 @@ BEGIN
     p_retval := 0;
     p_errmsg := '';
 
-    -- Validate instance ID
     IF p_instance_id IS NULL OR p_instance_id <= 0 THEN
         p_retval := 1;
         p_errmsg := 'Invalid instance ID';
         RETURN;
     END IF;
 
-    -- Update ping timestamp
     IF p_jobs_handled IS NOT NULL THEN
-        UPDATE broker_queueinstance
+        UPDATE broker.broker_queueinstance
-        SET last_ping_at = NOW(),
+        SET last_ping_at = NOW(), jobs_handled = p_jobs_handled
-            jobs_handled = p_jobs_handled
         WHERE id_broker_queueinstance = p_instance_id;
     ELSE
-        UPDATE broker_queueinstance
+        UPDATE broker.broker_queueinstance
         SET last_ping_at = NOW()
         WHERE id_broker_queueinstance = p_instance_id;
     END IF;
 
-    -- Check if instance was found
     IF NOT FOUND THEN
         p_retval := 2;
         p_errmsg := 'Instance not found';
@@ -49,11 +43,7 @@ EXCEPTION
 END;
 $$;
 
--- broker_shutdown_instance function
+CREATE OR REPLACE FUNCTION broker.broker_shutdown_instance(
--- Marks a broker instance as shutdown
--- Returns: p_retval (0=success, >0=error), p_errmsg (error message)
-
-CREATE OR REPLACE FUNCTION broker_shutdown_instance(
     p_instance_id BIGINT,
     OUT p_retval INTEGER,
     OUT p_errmsg TEXT
@@ -65,20 +55,16 @@ BEGIN
     p_retval := 0;
     p_errmsg := '';
 
-    -- Validate instance ID
     IF p_instance_id IS NULL OR p_instance_id <= 0 THEN
         p_retval := 1;
         p_errmsg := 'Invalid instance ID';
         RETURN;
     END IF;
 
-    -- Update instance status
+    UPDATE broker.broker_queueinstance
-    UPDATE broker_queueinstance
+    SET status = 'shutdown', shutdown_at = NOW()
-    SET status = 'shutdown',
-        shutdown_at = NOW()
     WHERE id_broker_queueinstance = p_instance_id;
 
-    -- Check if instance was found
     IF NOT FOUND THEN
         p_retval := 2;
         p_errmsg := 'Instance not found';
@@ -93,6 +79,5 @@ EXCEPTION
 END;
 $$;
 
--- Comments
+COMMENT ON FUNCTION broker.broker_ping_instance IS 'Updates the last ping timestamp for an instance';
-COMMENT ON FUNCTION broker_ping_instance IS 'Updates the last ping timestamp for an instance';
+COMMENT ON FUNCTION broker.broker_shutdown_instance IS 'Marks an instance as shutdown (does not release the advisory lock -- caller must pg_advisory_unlock on its pinned connection)';
-COMMENT ON FUNCTION broker_shutdown_instance IS 'Marks an instance as shutdown';

pkg/broker/install/sql/migrations/0013_broker_recover_stale_jobs.sql

@@ -0,0 +1,66 @@
+-- broker.broker_recover_stale_jobs
+-- Recovers jobs whose lease has expired while still 'running' (a worker died
+-- or was killed mid-execution without updating status). Applies the same
+-- retry/backoff rule as broker_run: retry while attempts remain, otherwise
+-- dead-letter. Marked SECURITY DEFINER so the sweep runs across all tenants
+-- regardless of caller: this requires the function's owner (whichever role
+-- runs the migrations, intended to be broker_admin) to have BYPASSRLS --
+-- broker_jobs/broker_job_dependency use FORCE ROW LEVEL SECURITY, so without
+-- BYPASSRLS on the owner this would silently only ever see the empty/no
+-- tenant context.
+
+CREATE OR REPLACE FUNCTION broker.broker_recover_stale_jobs(
+    OUT p_retval INTEGER,
+    OUT p_errmsg TEXT,
+    OUT p_recovered_count INTEGER
+)
+RETURNS RECORD
+LANGUAGE plpgsql
+SECURITY DEFINER
+SET search_path = broker, pg_temp
+AS $$
+DECLARE
+    v_backoff_base CONSTANT INTEGER := 5;
+    v_backoff_cap CONSTANT INTEGER := 300;
+BEGIN
+    p_retval := 0;
+    p_errmsg := '';
+    p_recovered_count := 0;
+
+    WITH stale AS (
+        SELECT id_broker_jobs, attempt_count, max_attempts
+        FROM broker.broker_jobs
+        WHERE complete_status = 1
+          AND lease_expires_at IS NOT NULL
+          AND lease_expires_at < NOW()
+        FOR UPDATE SKIP LOCKED
+    ),
+    recovered AS (
+        UPDATE broker.broker_jobs j
+        SET complete_status = CASE WHEN s.attempt_count < s.max_attempts THEN 0 ELSE 3 END,
+            available_at = CASE
+                WHEN s.attempt_count < s.max_attempts
+                THEN NOW() + make_interval(secs => LEAST(POWER(2, s.attempt_count)::INTEGER * v_backoff_base, v_backoff_cap))
+                ELSE j.available_at
+            END,
+            error_msg = CASE WHEN s.attempt_count >= s.max_attempts THEN COALESCE(j.error_msg, 'Lease expired and max attempts exhausted') ELSE j.error_msg END,
+            completed_at = CASE WHEN s.attempt_count >= s.max_attempts THEN NOW() ELSE j.completed_at END,
+            lease_token = NULL,
+            leased_at = NULL,
+            lease_expires_at = NULL,
+            updated_at = NOW()
+        FROM stale s
+        WHERE j.id_broker_jobs = s.id_broker_jobs
+        RETURNING j.id_broker_jobs
+    )
+    SELECT COUNT(*) INTO p_recovered_count FROM recovered;
+
+EXCEPTION
+    WHEN OTHERS THEN
+        p_retval := 99;
+        p_errmsg := SQLERRM;
+        RAISE WARNING 'broker_recover_stale_jobs error: %', SQLERRM;
+END;
+$$;
+
+COMMENT ON FUNCTION broker.broker_recover_stale_jobs IS 'Requeues (or dead-letters) jobs whose lease expired while still running';

pkg/broker/install/sql/migrations/0014_job_groups.sql

@@ -0,0 +1,41 @@
+-- Adds job groups: every job belongs to a job_group (defaults to its own
+-- job_name when not given explicitly, set by broker_add_job). A dependency
+-- can now target a whole group instead of a single job id -- the dependent
+-- job is claimable once every job tagged with that group has completed
+-- (complete_status = 2); already-completed group members simply drop out of
+-- the gating check, they don't need to have existed at any particular time.
+-- The existing id-based dependency (broker_job_dependency.depends_on_job_id)
+-- is kept as-is; each dependency row targets exactly one of an id or a group.
+
+ALTER TABLE broker.broker_jobs ADD COLUMN IF NOT EXISTS job_group TEXT;
+UPDATE broker.broker_jobs SET job_group = job_name WHERE job_group IS NULL;
+ALTER TABLE broker.broker_jobs ALTER COLUMN job_group SET NOT NULL;
+
+CREATE INDEX IF NOT EXISTS idx_broker_jobs_group
+    ON broker.broker_jobs (tenant_id, job_group, complete_status);
+
+ALTER TABLE broker.broker_job_dependency DROP CONSTRAINT IF EXISTS broker_job_dependency_pkey;
+ALTER TABLE broker.broker_job_dependency ALTER COLUMN depends_on_job_id DROP NOT NULL;
+ALTER TABLE broker.broker_job_dependency ADD COLUMN IF NOT EXISTS depends_on_group TEXT;
+ALTER TABLE broker.broker_job_dependency ADD COLUMN IF NOT EXISTS id_broker_job_dependency BIGSERIAL;
+ALTER TABLE broker.broker_job_dependency ADD PRIMARY KEY (id_broker_job_dependency);
+
+ALTER TABLE broker.broker_job_dependency DROP CONSTRAINT IF EXISTS broker_job_dependency_one_target;
+ALTER TABLE broker.broker_job_dependency ADD CONSTRAINT broker_job_dependency_one_target CHECK (
+    (depends_on_job_id IS NOT NULL AND depends_on_group IS NULL) OR
+    (depends_on_job_id IS NULL AND depends_on_group IS NOT NULL)
+);
+
+-- Plain (non-partial) unique constraint, matching the old PK's guarantee --
+-- NULLs in depends_on_job_id (the group-dependency rows) are never
+-- considered equal by a standard unique constraint, so this only constrains
+-- id-based rows, and keeps "ON CONFLICT (job_id, depends_on_job_id)" (no
+-- predicate needed) working for existing callers.
+ALTER TABLE broker.broker_job_dependency DROP CONSTRAINT IF EXISTS broker_job_dependency_unique_id;
+ALTER TABLE broker.broker_job_dependency ADD CONSTRAINT broker_job_dependency_unique_id
+    UNIQUE (job_id, depends_on_job_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_broker_job_dependency_unique_group
+    ON broker.broker_job_dependency (job_id, depends_on_group) WHERE depends_on_group IS NOT NULL;
+
+COMMENT ON COLUMN broker.broker_jobs.job_group IS 'Group tag for this job; defaults to job_name. Other jobs can depend on the whole group.';
+COMMENT ON COLUMN broker.broker_job_dependency.depends_on_group IS 'Alternative to depends_on_job_id: job_id is not claimable until every job with job_group = depends_on_group has completed';

pkg/broker/install/sql/migrations/0015_broker_add_job_groups.sql

@@ -0,0 +1,184 @@
+-- broker.broker_add_job: adds job groups.
+-- p_job_group defaults to p_job_name when not given. p_depends_on_groups is
+-- the group-based counterpart to the existing p_depends_on_job_ids: the new
+-- job is not claimable until every job tagged with each named group has
+-- completed. Both dependency kinds can be combined on the same job.
+-- New parameters are appended after existing ones with defaults, so every
+-- existing positional call (however many args it passes) keeps working
+-- unchanged. Appending arguments changes the function's identity though --
+-- CREATE OR REPLACE would create a second, ambiguous overload rather than
+-- replacing -- so the old 10-arg signature is dropped explicitly first.
+
+DROP FUNCTION IF EXISTS broker.broker_add_job(
+    TEXT, TEXT, INTEGER, INTEGER, TEXT, TEXT, BIGINT, BIGINT[], TEXT, INTEGER
+);
+
+CREATE OR REPLACE FUNCTION broker.broker_add_job(
+    p_job_name TEXT,
+    p_execute_str TEXT,
+    p_job_queue INTEGER DEFAULT 1,
+    p_job_priority INTEGER DEFAULT 0,
+    p_job_language TEXT DEFAULT 'sql',
+    p_run_as TEXT DEFAULT NULL,
+    p_schedule_id BIGINT DEFAULT NULL,
+    p_depends_on_job_ids BIGINT[] DEFAULT NULL,
+    p_idempotency_key TEXT DEFAULT NULL,
+    p_max_attempts INTEGER DEFAULT 1,
+    p_job_group TEXT DEFAULT NULL,
+    p_depends_on_groups TEXT[] DEFAULT NULL,
+    OUT p_retval INTEGER,
+    OUT p_errmsg TEXT,
+    OUT p_job_id BIGINT
+)
+RETURNS RECORD
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    v_notification_payload JSON;
+    v_tenant_id TEXT;
+    v_job_group TEXT;
+    v_dep_id BIGINT;
+    v_dep_group TEXT;
+    v_cycle_exists BOOLEAN;
+BEGIN
+    p_retval := 0;
+    p_errmsg := '';
+    p_job_id := NULL;
+
+    IF p_job_name IS NULL OR p_job_name = '' THEN
+        p_retval := 1;
+        p_errmsg := 'Job name is required';
+        RETURN;
+    END IF;
+
+    IF p_execute_str IS NULL OR p_execute_str = '' THEN
+        p_retval := 2;
+        p_errmsg := 'Execute string is required';
+        RETURN;
+    END IF;
+
+    IF p_job_queue IS NULL OR p_job_queue <= 0 THEN
+        p_retval := 3;
+        p_errmsg := 'Invalid job queue number';
+        RETURN;
+    END IF;
+
+    IF p_max_attempts IS NULL OR p_max_attempts <= 0 THEN
+        p_max_attempts := 1;
+    END IF;
+
+    v_job_group := COALESCE(NULLIF(p_job_group, ''), p_job_name);
+
+    -- Falls back to 'default' when the caller never called broker_set_tenant,
+    -- so single-tenant use (and the RLS WITH CHECK on insert) keeps working.
+    v_tenant_id := COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default');
+
+    IF p_idempotency_key IS NOT NULL THEN
+        SELECT id_broker_jobs INTO p_job_id
+        FROM broker.broker_jobs
+        WHERE job_queue = p_job_queue
+          AND idempotency_key = p_idempotency_key
+          AND tenant_id = v_tenant_id;
+
+        IF FOUND THEN
+            p_errmsg := 'Job with this idempotency key already exists; returning existing job id';
+            RETURN;
+        END IF;
+    END IF;
+
+    INSERT INTO broker.broker_jobs (
+        job_name, job_group, job_priority, job_queue, job_language, execute_str, run_as,
+        rid_broker_schedule, tenant_id, max_attempts, idempotency_key, complete_status
+    ) VALUES (
+        p_job_name, v_job_group, p_job_priority, p_job_queue, p_job_language, p_execute_str, p_run_as,
+        p_schedule_id, v_tenant_id, p_max_attempts, p_idempotency_key, 0
+    )
+    RETURNING id_broker_jobs INTO p_job_id;
+
+    IF p_depends_on_job_ids IS NOT NULL THEN
+        FOREACH v_dep_id IN ARRAY p_depends_on_job_ids LOOP
+            IF v_dep_id IS NULL THEN
+                CONTINUE;
+            END IF;
+
+            IF v_dep_id = p_job_id THEN
+                p_retval := 20;
+                p_errmsg := 'Invalid dependency: a job cannot depend on itself';
+                RETURN;
+            END IF;
+
+            SELECT EXISTS (
+                SELECT 1 FROM broker.broker_job_dependency
+                WHERE job_id = v_dep_id AND depends_on_job_id = p_job_id
+            ) INTO v_cycle_exists;
+
+            IF v_cycle_exists THEN
+                p_retval := 21;
+                p_errmsg := format('Invalid dependency: job %s already depends on %s (would create a cycle)', v_dep_id, p_job_id);
+                RETURN;
+            END IF;
+
+            INSERT INTO broker.broker_job_dependency (job_id, depends_on_job_id, tenant_id)
+            VALUES (p_job_id, v_dep_id, v_tenant_id)
+            ON CONFLICT (job_id, depends_on_job_id) DO NOTHING;
+        END LOOP;
+    END IF;
+
+    IF p_depends_on_groups IS NOT NULL THEN
+        FOREACH v_dep_group IN ARRAY p_depends_on_groups LOOP
+            IF v_dep_group IS NULL OR v_dep_group = '' THEN
+                CONTINUE;
+            END IF;
+
+            IF v_dep_group = v_job_group THEN
+                p_retval := 22;
+                p_errmsg := format('Invalid dependency: a job cannot depend on its own group (%s)', v_job_group);
+                RETURN;
+            END IF;
+
+            SELECT EXISTS (
+                SELECT 1
+                FROM broker.broker_jobs j
+                JOIN broker.broker_job_dependency d ON d.job_id = j.id_broker_jobs
+                WHERE j.tenant_id = v_tenant_id
+                  AND j.job_group = v_dep_group
+                  AND d.depends_on_group = v_job_group
+            ) INTO v_cycle_exists;
+
+            IF v_cycle_exists THEN
+                p_retval := 23;
+                p_errmsg := format('Invalid dependency: group %s already depends on %s (would create a cycle)', v_dep_group, v_job_group);
+                RETURN;
+            END IF;
+
+            INSERT INTO broker.broker_job_dependency (job_id, depends_on_group, tenant_id)
+            VALUES (p_job_id, v_dep_group, v_tenant_id)
+            ON CONFLICT (job_id, depends_on_group) WHERE depends_on_group IS NOT NULL DO NOTHING;
+        END LOOP;
+    END IF;
+
+    v_notification_payload := json_build_object(
+        'queue', p_job_queue,
+        'job_id', p_job_id
+    );
+
+    PERFORM pg_notify('broker.event', v_notification_payload::text);
+
+EXCEPTION
+    WHEN unique_violation THEN
+        -- Concurrent insert raced us to the same idempotency key.
+        p_retval := 0;
+        p_errmsg := 'Job with this idempotency key already exists; returning existing job id';
+        SELECT id_broker_jobs INTO p_job_id
+        FROM broker.broker_jobs
+        WHERE job_queue = p_job_queue
+          AND idempotency_key = p_idempotency_key
+          AND tenant_id = v_tenant_id;
+    WHEN OTHERS THEN
+        p_retval := 99;
+        p_errmsg := SQLERRM;
+        RAISE WARNING 'broker_add_job error: %', SQLERRM;
+END;
+$$;
+
+COMMENT ON FUNCTION broker.broker_add_job IS 'Adds a job (with optional id/group dependencies, job group, and idempotency key) and sends a wake-only NOTIFY';

pkg/broker/install/sql/migrations/0016_broker_get_groups.sql

@@ -0,0 +1,91 @@
+-- broker.broker_get: also gates claiming on group-based dependencies
+-- (broker_job_dependency.depends_on_group) alongside the existing id-based
+-- ones. Signature is unchanged, only the eligibility query grows a second
+-- NOT EXISTS clause.
+
+CREATE OR REPLACE FUNCTION broker.broker_get(
+    p_queue_number INTEGER,
+    p_instance_id BIGINT DEFAULT NULL,
+    p_lease_seconds INTEGER DEFAULT 60,
+    OUT p_retval INTEGER,
+    OUT p_errmsg TEXT,
+    OUT p_job_id BIGINT,
+    OUT p_lease_token UUID
+)
+RETURNS RECORD
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    v_job_id BIGINT;
+    v_lease_token UUID;
+BEGIN
+    p_retval := 0;
+    p_errmsg := '';
+    p_job_id := NULL;
+    p_lease_token := NULL;
+
+    IF p_queue_number IS NULL OR p_queue_number <= 0 THEN
+        p_retval := 1;
+        p_errmsg := 'Invalid queue number';
+        RETURN;
+    END IF;
+
+    IF p_lease_seconds IS NULL OR p_lease_seconds <= 0 THEN
+        p_lease_seconds := 60;
+    END IF;
+
+    SELECT candidate.id_broker_jobs
+    INTO v_job_id
+    FROM broker.broker_jobs candidate
+    WHERE candidate.job_queue = p_queue_number
+      AND candidate.complete_status = 0
+      AND candidate.available_at <= NOW()
+      AND NOT EXISTS (
+        SELECT 1
+        FROM broker.broker_job_dependency d
+        JOIN broker.broker_jobs dep ON dep.id_broker_jobs = d.depends_on_job_id
+        WHERE d.job_id = candidate.id_broker_jobs
+          AND dep.complete_status <> 2
+      )
+      AND NOT EXISTS (
+        SELECT 1
+        FROM broker.broker_job_dependency d
+        JOIN broker.broker_jobs dep ON dep.tenant_id = candidate.tenant_id
+                                    AND dep.job_group = d.depends_on_group
+        WHERE d.job_id = candidate.id_broker_jobs
+          AND dep.id_broker_jobs <> candidate.id_broker_jobs
+          AND dep.complete_status <> 2
+      )
+    ORDER BY candidate.job_priority DESC, candidate.created_at ASC, candidate.id_broker_jobs ASC
+    LIMIT 1
+    FOR UPDATE SKIP LOCKED;
+
+    IF NOT FOUND THEN
+        RETURN;
+    END IF;
+
+    v_lease_token := gen_random_uuid();
+
+    UPDATE broker.broker_jobs
+    SET complete_status = 1,  -- running
+        started_at = NOW(),
+        rid_broker_queueinstance = p_instance_id,
+        attempt_count = attempt_count + 1,
+        lease_token = v_lease_token,
+        leased_at = NOW(),
+        lease_expires_at = NOW() + make_interval(secs => p_lease_seconds),
+        updated_at = NOW()
+    WHERE id_broker_jobs = v_job_id;
+
+    p_job_id := v_job_id;
+    p_lease_token := v_lease_token;
+
+EXCEPTION
+    WHEN OTHERS THEN
+        p_retval := 2;
+        p_errmsg := SQLERRM;
+        RAISE WARNING 'broker_get error: %', SQLERRM;
+END;
+$$;
+
+COMMENT ON FUNCTION broker.broker_get IS 'Claims the next eligible job from a queue (honoring id- and group-based dependencies) and grants a lease';

pkg/broker/install/sql/migrations/0017_broker_add_job_simple_groups.sql

@@ -0,0 +1,41 @@
+-- broker.broker_add_job_simple
+-- Convenience wrapper around broker.broker_add_job for the common case: a
+-- job named p_job_name running p_execute_str at p_job_priority, depending on
+-- other jobs by group name (p_depends_on_groups) -- every job's group
+-- defaults to its own job_name, so passing job names here just works.
+-- Pure pass-through: no name-to-id resolution needed since dependencies are
+-- resolved live, by group, inside broker_get.
+
+CREATE OR REPLACE FUNCTION broker.broker_add_job_simple(
+    p_job_name TEXT,
+    p_execute_str TEXT,
+    p_job_priority INTEGER DEFAULT 0,
+    p_depends_on_groups TEXT[] DEFAULT NULL,
+    OUT p_retval INTEGER,
+    OUT p_errmsg TEXT,
+    OUT p_job_id BIGINT
+)
+RETURNS RECORD
+LANGUAGE plpgsql
+AS $$
+BEGIN
+    SELECT r.p_retval, r.p_errmsg, r.p_job_id
+    INTO p_retval, p_errmsg, p_job_id
+    FROM broker.broker_add_job(
+        p_job_name,
+        p_execute_str,
+        1,                    -- p_job_queue
+        p_job_priority,
+        'sql',                -- p_job_language
+        NULL,                 -- p_run_as
+        NULL,                 -- p_schedule_id
+        NULL,                 -- p_depends_on_job_ids
+        NULL,                 -- p_idempotency_key
+        1,                    -- p_max_attempts
+        NULL,                 -- p_job_group (defaults to p_job_name)
+        p_depends_on_groups
+    ) AS r;
+END;
+$$;
+
+COMMENT ON FUNCTION broker.broker_add_job_simple IS 'Shortcut for broker_add_job: name, execute string, priority, and dependencies by group name (defaults to job name)';

pkg/broker/install/sql/procedures/00_install.sql

@@ -1,13 +0,0 @@
--- PostgreSQL Broker Procedures Installation Script
--- Run this script to create all required stored procedures
-
-\echo 'Installing PostgreSQL Broker procedures...'
-
-\i 01_broker_get.sql
-\i 02_broker_run.sql
-\i 03_broker_set.sql
-\i 04_broker_register_instance.sql
-\i 05_broker_add_job.sql
-\i 06_broker_ping_instance.sql
-
-\echo 'PostgreSQL Broker procedures installed successfully!'

pkg/broker/install/sql/procedures/01_broker_get.sql

@@ -1,76 +0,0 @@
--- broker_get function
--- Fetches the next job from the queue for a given queue number
--- Returns: p_retval (0=success, >0=error), p_errmsg (error message), p_job_id (job ID if found)
-
-CREATE OR REPLACE FUNCTION broker_get(
-    p_queue_number INTEGER,
-    p_instance_id BIGINT DEFAULT NULL,
-    OUT p_retval INTEGER,
-    OUT p_errmsg TEXT,
-    OUT p_job_id BIGINT
-)
-RETURNS RECORD
-LANGUAGE plpgsql
-AS $$
-DECLARE
-    v_job_record RECORD;
-BEGIN
-    p_retval := 0;
-    p_errmsg := '';
-    p_job_id := NULL;
-
-    -- Validate queue number
-    IF p_queue_number IS NULL OR p_queue_number <= 0 THEN
-        p_retval := 1;
-        p_errmsg := 'Invalid queue number';
-        RETURN;
-    END IF;
-
-    -- Find and lock the next pending job for this queue
-    -- Uses SKIP LOCKED to avoid blocking on jobs being processed by other workers
-    -- Skip jobs with pending dependencies
-    SELECT id_broker_jobs, job_name, job_priority, execute_str
-    INTO v_job_record
-    FROM broker_jobs
-    WHERE job_queue = p_queue_number
-      AND complete_status = 0  -- pending
-      AND (
-        depends_on IS NULL  -- no dependencies
-        OR depends_on = '{}'  -- empty dependencies
-        OR NOT EXISTS (  -- all dependencies completed
-          SELECT 1
-          FROM broker_jobs dep
-          WHERE dep.job_name = ANY(broker_jobs.depends_on)
-            AND dep.complete_status = 0  -- pending dependency
-        )
-      )
-    ORDER BY job_priority DESC, created_at ASC
-    LIMIT 1
-    FOR UPDATE SKIP LOCKED;
-
-    -- If no job found, return success with NULL job_id
-    IF NOT FOUND THEN
-        RETURN;
-    END IF;
-
-    -- Update job status to running
-    UPDATE broker_jobs
-    SET complete_status = 1,  -- running
-        started_at = NOW(),
-        rid_broker_queueinstance = p_instance_id,
-        updated_at = NOW()
-    WHERE id_broker_jobs = v_job_record.id_broker_jobs;
-
-    -- Return the job ID
-    p_job_id := v_job_record.id_broker_jobs;
-
-EXCEPTION
-    WHEN OTHERS THEN
-        p_retval := 2;
-        p_errmsg := SQLERRM;
-        RAISE WARNING 'broker_get error: %', SQLERRM;
-END;
-$$;
-
--- Comments
-COMMENT ON FUNCTION broker_get IS 'Fetches the next pending job from the specified queue';

pkg/broker/install/sql/procedures/02_broker_run.sql

@@ -1,113 +0,0 @@
--- broker_run function
--- Executes a job by its ID
--- Returns: p_retval (0=success, >0=error), p_errmsg (error message)
-
-CREATE OR REPLACE FUNCTION broker_run(
-    p_job_id BIGINT,
-    OUT p_retval INTEGER,
-    OUT p_errmsg TEXT
-)
-RETURNS RECORD
-LANGUAGE plpgsql
-AS $$
-DECLARE
-    v_job_record RECORD;
-    v_execute_result TEXT;
-    v_error_occurred BOOLEAN := false;
-BEGIN
-    p_retval := 0;
-    p_errmsg := '';
-    v_execute_result := '';
-
-    -- Validate job ID
-    IF p_job_id IS NULL OR p_job_id <= 0 THEN
-        p_retval := 1;
-        p_errmsg := 'Invalid job ID';
-        RETURN;
-    END IF;
-
-    -- Get job details
-    SELECT id_broker_jobs, execute_str, job_language, run_as, complete_status
-    INTO v_job_record
-    FROM broker_jobs
-    WHERE id_broker_jobs = p_job_id
-    FOR UPDATE;
-
-    -- Check if job exists
-    IF NOT FOUND THEN
-        p_retval := 2;
-        p_errmsg := 'Job not found';
-        RETURN;
-    END IF;
-
-    -- Check if job is in running state
-    IF v_job_record.complete_status != 1 THEN
-        p_retval := 3;
-        p_errmsg := format('Job is not in running state (status: %s)', v_job_record.complete_status);
-        RETURN;
-    END IF;
-
-    -- Execute the job
-    BEGIN
-        -- For SQL/PLPGSQL jobs, execute directly
-        IF v_job_record.job_language IN ('sql', 'plpgsql') THEN
-            EXECUTE v_job_record.execute_str;
-            v_execute_result := 'Success';
-        ELSE
-            -- Other languages would need external execution
-            p_retval := 4;
-            p_errmsg := format('Unsupported job language: %s', v_job_record.job_language);
-            v_error_occurred := true;
-        END IF;
-
-    EXCEPTION
-        WHEN OTHERS THEN
-            v_error_occurred := true;
-            p_retval := 5;
-            p_errmsg := SQLERRM;
-            v_execute_result := format('Error: %s', SQLERRM);
-    END;
-
-    -- Update job with results
-    IF v_error_occurred THEN
-        UPDATE broker_jobs
-        SET complete_status = 3,  -- failed
-            error_msg = p_errmsg,
-            execute_result = v_execute_result,
-            completed_at = NOW(),
-            updated_at = NOW()
-        WHERE id_broker_jobs = p_job_id;
-    ELSE
-        UPDATE broker_jobs
-        SET complete_status = 2,  -- completed
-            execute_result = v_execute_result,
-            error_msg = NULL,
-            completed_at = NOW(),
-            updated_at = NOW()
-        WHERE id_broker_jobs = p_job_id;
-    END IF;
-
-EXCEPTION
-    WHEN OTHERS THEN
-        p_retval := 6;
-        p_errmsg := SQLERRM;
-        RAISE WARNING 'broker_run error: %', SQLERRM;
-
-        -- Try to update job status to failed
-        BEGIN
-            UPDATE broker_jobs
-            SET complete_status = 3,  -- failed
-                error_msg = SQLERRM,
-                completed_at = NOW(),
-                updated_at = NOW()
-            WHERE id_broker_jobs = p_job_id;
-        EXCEPTION
-            WHEN OTHERS THEN
-                -- Ignore update errors
-                NULL;
-        END;
-END;
-$$;
-
--- Comments
-COMMENT ON FUNCTION broker_run IS 'Executes a job by its ID and updates the status';

pkg/broker/install/sql/procedures/03_broker_set.sql

@@ -1,95 +0,0 @@
--- broker_set function
--- Sets broker runtime options and context
--- Supports: user, application_name, and custom settings
--- Returns: p_retval (0=success, >0=error), p_errmsg (error message)
-
-CREATE OR REPLACE FUNCTION broker_set(
-    p_option_name TEXT,
-    p_option_value TEXT,
-    OUT p_retval INTEGER,
-    OUT p_errmsg TEXT
-)
-RETURNS RECORD
-LANGUAGE plpgsql
-AS $$
-DECLARE
-    v_sql TEXT;
-BEGIN
-    p_retval := 0;
-    p_errmsg := '';
-
-    -- Validate inputs
-    IF p_option_name IS NULL OR p_option_name = '' THEN
-        p_retval := 1;
-        p_errmsg := 'Option name is required';
-        RETURN;
-    END IF;
-
-    -- Handle different option types
-    CASE LOWER(p_option_name)
-        WHEN 'user' THEN
-            -- Set session user context
-            -- This is useful for audit trails and permissions
-            BEGIN
-                v_sql := format('SET SESSION AUTHORIZATION %I', p_option_value);
-                EXECUTE v_sql;
-            EXCEPTION
-                WHEN OTHERS THEN
-                    p_retval := 2;
-                    p_errmsg := format('Failed to set user: %s', SQLERRM);
-                    RETURN;
-            END;
-
-        WHEN 'application_name' THEN
-            -- Set application name (visible in pg_stat_activity)
-            BEGIN
-                v_sql := format('SET application_name TO %L', p_option_value);
-                EXECUTE v_sql;
-            EXCEPTION
-                WHEN OTHERS THEN
-                    p_retval := 3;
-                    p_errmsg := format('Failed to set application_name: %s', SQLERRM);
-                    RETURN;
-            END;
-
-        WHEN 'search_path' THEN
-            -- Set schema search path
-            BEGIN
-                v_sql := format('SET search_path TO %s', p_option_value);
-                EXECUTE v_sql;
-            EXCEPTION
-                WHEN OTHERS THEN
-                    p_retval := 4;
-                    p_errmsg := format('Failed to set search_path: %s', SQLERRM);
-                    RETURN;
-            END;
-
-        WHEN 'timezone' THEN
-            -- Set timezone
-            BEGIN
-                v_sql := format('SET timezone TO %L', p_option_value);
-                EXECUTE v_sql;
-            EXCEPTION
-                WHEN OTHERS THEN
-                    p_retval := 5;
-                    p_errmsg := format('Failed to set timezone: %s', SQLERRM);
-                    RETURN;
-            END;
-
-        ELSE
-            -- Unknown option
-            p_retval := 10;
-            p_errmsg := format('Unknown option: %s', p_option_name);
-            RETURN;
-    END CASE;
-
-EXCEPTION
-    WHEN OTHERS THEN
-        p_retval := 99;
-        p_errmsg := SQLERRM;
-        RAISE WARNING 'broker_set error: %', SQLERRM;
-END;
-$$;
-
--- Comments
-COMMENT ON FUNCTION broker_set IS 'Sets broker runtime options and session context (user, application_name, search_path, timezone)';

pkg/broker/install/sql/procedures/04_broker_register_instance.sql

@@ -1,82 +0,0 @@
--- broker_register_instance function
--- Registers a new broker instance in the database
--- Returns: p_retval (0=success, >0=error), p_errmsg (error message), p_instance_id (new instance ID)
-
-CREATE OR REPLACE FUNCTION broker_register_instance(
-    p_name TEXT,
-    p_hostname TEXT,
-    p_pid INTEGER,
-    p_version TEXT,
-    p_queue_count INTEGER,
-    OUT p_retval INTEGER,
-    OUT p_errmsg TEXT,
-    OUT p_instance_id BIGINT
-)
-RETURNS RECORD
-LANGUAGE plpgsql
-AS $$
-DECLARE
-    v_active_count INTEGER;
-BEGIN
-    p_retval := 0;
-    p_errmsg := '';
-    p_instance_id := NULL;
-
-    -- Validate inputs
-    IF p_name IS NULL OR p_name = '' THEN
-        p_retval := 1;
-        p_errmsg := 'Instance name is required';
-        RETURN;
-    END IF;
-
-    IF p_hostname IS NULL OR p_hostname = '' THEN
-        p_retval := 2;
-        p_errmsg := 'Hostname is required';
-        RETURN;
-    END IF;
-
-    -- Check for existing active instances
-    -- Only one broker instance should be active per database
-    SELECT COUNT(*)
-    INTO v_active_count
-    FROM broker_queueinstance
-    WHERE status = 'active';
-
-    IF v_active_count > 0 THEN
-        p_retval := 3;
-        p_errmsg := 'Another broker instance is already active in this database. Only one broker instance per database is allowed.';
-        RETURN;
-    END IF;
-
-    -- Insert new instance
-    INSERT INTO broker_queueinstance (
-        name,
-        hostname,
-        pid,
-        version,
-        status,
-        queue_count,
-        started_at,
-        last_ping_at
-    ) VALUES (
-        p_name,
-        p_hostname,
-        p_pid,
-        p_version,
-        'active',
-        p_queue_count,
-        NOW(),
-        NOW()
-    )
-    RETURNING id_broker_queueinstance INTO p_instance_id;
-
-EXCEPTION
-    WHEN OTHERS THEN
-        p_retval := 99;
-        p_errmsg := SQLERRM;
-        RAISE WARNING 'broker_register_instance error: %', SQLERRM;
-END;
-$$;
-
--- Comments
-COMMENT ON FUNCTION broker_register_instance IS 'Registers a new broker instance';

pkg/broker/install/sql/procedures/05_broker_add_job.sql

@@ -1,91 +0,0 @@
--- broker_add_job function
--- Adds a new job to the broker queue and sends a notification
--- Returns: p_retval (0=success, >0=error), p_errmsg (error message), p_job_id (new job ID)
-
-CREATE OR REPLACE FUNCTION broker_add_job(
-    p_job_name TEXT,
-    p_execute_str TEXT,
-    p_job_queue INTEGER DEFAULT 1,
-    p_job_priority INTEGER DEFAULT 0,
-    p_job_language TEXT DEFAULT 'sql',
-    p_run_as TEXT DEFAULT NULL,
-    p_schedule_id BIGINT DEFAULT NULL,
-    p_depends_on TEXT[] DEFAULT NULL,
-    OUT p_retval INTEGER,
-    OUT p_errmsg TEXT,
-    OUT p_job_id BIGINT
-)
-RETURNS RECORD
-LANGUAGE plpgsql
-AS $$
-DECLARE
-    v_notification_payload JSON;
-BEGIN
-    p_retval := 0;
-    p_errmsg := '';
-    p_job_id := NULL;
-
-    -- Validate inputs
-    IF p_job_name IS NULL OR p_job_name = '' THEN
-        p_retval := 1;
-        p_errmsg := 'Job name is required';
-        RETURN;
-    END IF;
-
-    IF p_execute_str IS NULL OR p_execute_str = '' THEN
-        p_retval := 2;
-        p_errmsg := 'Execute string is required';
-        RETURN;
-    END IF;
-
-    IF p_job_queue IS NULL OR p_job_queue <= 0 THEN
-        p_retval := 3;
-        p_errmsg := 'Invalid job queue number';
-        RETURN;
-    END IF;
-
-    -- Insert new job
-    INSERT INTO broker_jobs (
-        job_name,
-        job_priority,
-        job_queue,
-        job_language,
-        execute_str,
-        run_as,
-        rid_broker_schedule,
-        depends_on,
-        complete_status
-    ) VALUES (
-        p_job_name,
-        p_job_priority,
-        p_job_queue,
-        p_job_language,
-        p_execute_str,
-        p_run_as,
-        p_schedule_id,
-        p_depends_on,
-        0  -- pending
-    )
-    RETURNING id_broker_jobs INTO p_job_id;
-
-    -- Create notification payload
-    v_notification_payload := json_build_object(
-        'id', p_job_id,
-        'job_name', p_job_name,
-        'job_queue', p_job_queue,
-        'job_priority', p_job_priority
-    );
-
-    -- Send notification to broker
-    PERFORM pg_notify('broker.event', v_notification_payload::text);
-
-EXCEPTION
-    WHEN OTHERS THEN
-        p_retval := 99;
-        p_errmsg := SQLERRM;
-        RAISE WARNING 'broker_add_job error: %', SQLERRM;
-END;
-$$;
-
--- Comments
-COMMENT ON FUNCTION broker_add_job IS 'Adds a new job to the broker queue and sends a NOTIFY event';

pkg/broker/install/sql/roles/0001_roles.sql

@@ -0,0 +1,60 @@
+-- Reference role/grant setup for pgsql-broker.
+--
+-- Applied via `pgsql-broker install --with-roles` (run once per cluster;
+-- the schema/grant statements are safe to re-run per database). The
+-- __BROKER_*_PASSWORD__ placeholders are substituted by the installer at
+-- render time -- never edit this file to hardcode a real password. Each
+-- CREATE ROLE is guarded so re-running this (e.g. against a second
+-- configured database) rotates the password via ALTER ROLE instead of
+-- failing on an already-existing role.
+--
+-- Roles:
+--   broker_admin    -- schema owner, runs migrations (`pgsql-broker install`).
+--                      Needs BYPASSRLS so broker_recover_stale_jobs (SECURITY
+--                      DEFINER, owned by this role) can sweep all tenants.
+--   broker_runtime  -- the role the running broker process connects as.
+--                      No BYPASSRLS, no ownership, SEARCH_PATH=broker so the
+--                      broker's unqualified table/function references resolve.
+--   broker_enqueue  -- narrow role for services that only need to add jobs.
+
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'broker_admin') THEN
+        CREATE ROLE broker_admin LOGIN PASSWORD __BROKER_ADMIN_PASSWORD__ BYPASSRLS;
+    ELSE
+        ALTER ROLE broker_admin WITH LOGIN PASSWORD __BROKER_ADMIN_PASSWORD__ BYPASSRLS;
+    END IF;
+
+    IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'broker_runtime') THEN
+        CREATE ROLE broker_runtime LOGIN PASSWORD __BROKER_RUNTIME_PASSWORD__;
+    ELSE
+        ALTER ROLE broker_runtime WITH LOGIN PASSWORD __BROKER_RUNTIME_PASSWORD__;
+    END IF;
+
+    IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'broker_enqueue') THEN
+        CREATE ROLE broker_enqueue LOGIN PASSWORD __BROKER_ENQUEUE_PASSWORD__;
+    ELSE
+        ALTER ROLE broker_enqueue WITH LOGIN PASSWORD __BROKER_ENQUEUE_PASSWORD__;
+    END IF;
+END
+$$;
+
+ALTER ROLE broker_runtime SET search_path = broker, public;
+ALTER ROLE broker_enqueue SET search_path = broker, public;
+
+-- Run once broker.broker_jobs etc. already exist (i.e. after `pgsql-broker install`
+-- as broker_admin), so schema ownership/grants land on the right objects.
+
+ALTER SCHEMA broker OWNER TO broker_admin;
+GRANT USAGE ON SCHEMA broker TO broker_runtime, broker_enqueue;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA broker TO broker_runtime;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA broker TO broker_runtime;
+GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA broker TO broker_runtime;
+
+REVOKE ALL ON ALL FUNCTIONS IN SCHEMA broker FROM broker_enqueue;
+GRANT EXECUTE ON FUNCTION broker.broker_add_job TO broker_enqueue;
+GRANT EXECUTE ON FUNCTION broker.broker_add_job_simple TO broker_enqueue;
+GRANT EXECUTE ON FUNCTION broker.broker_set_tenant TO broker_enqueue;
+GRANT INSERT, SELECT ON broker.broker_jobs, broker.broker_job_dependency TO broker_enqueue;
+GRANT USAGE ON broker.broker_jobs_id_broker_jobs_seq TO broker_enqueue;

pkg/broker/install/sql/tables/00_install.sql

@@ -1,10 +0,0 @@
--- PostgreSQL Broker Tables Installation Script
--- Run this script to create all required tables
-
-\echo 'Installing PostgreSQL Broker tables...'
-
-\i 01_broker_queueinstance.sql
-\i 02_broker_schedule.sql
-\i 03_broker_jobs.sql
-
-\echo 'PostgreSQL Broker tables installed successfully!'

pkg/broker/install/sql/tables/01_broker_queueinstance.sql

@@ -1,31 +0,0 @@
--- broker_queueinstance table
--- Tracks active and historical broker queue instances
-
-CREATE TABLE IF NOT EXISTS broker_queueinstance (
-    id_broker_queueinstance BIGSERIAL PRIMARY KEY,
-    name VARCHAR(255) NOT NULL,
-    hostname VARCHAR(255) NOT NULL,
-    pid INTEGER NOT NULL,
-    version VARCHAR(50) NOT NULL,
-    status VARCHAR(20) NOT NULL DEFAULT 'active',
-    started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
-    last_ping_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
-    shutdown_at TIMESTAMP WITH TIME ZONE,
-    queue_count INTEGER NOT NULL DEFAULT 0,
-    jobs_handled BIGINT NOT NULL DEFAULT 0,
-
-    CONSTRAINT broker_queueinstance_status_check CHECK (status IN ('active', 'inactive', 'shutdown'))
-);
-
--- Indexes
-CREATE INDEX IF NOT EXISTS idx_broker_queueinstance_status ON broker_queueinstance(status);
-CREATE INDEX IF NOT EXISTS idx_broker_queueinstance_hostname ON broker_queueinstance(hostname);
-CREATE INDEX IF NOT EXISTS idx_broker_queueinstance_last_ping ON broker_queueinstance(last_ping_at);
-
--- Comments
-COMMENT ON TABLE broker_queueinstance IS 'Tracks broker queue instances (active and historical)';
-COMMENT ON COLUMN broker_queueinstance.name IS 'Human-readable name of the broker instance';
-COMMENT ON COLUMN broker_queueinstance.hostname IS 'Hostname where the broker is running';
-COMMENT ON COLUMN broker_queueinstance.pid IS 'Process ID of the broker';
-COMMENT ON COLUMN broker_queueinstance.status IS 'Current status: active, inactive, or shutdown';
-COMMENT ON COLUMN broker_queueinstance.jobs_handled IS 'Total number of jobs handled by this instance';

pkg/broker/install/sql/tables/02_broker_schedule.sql

@@ -1,50 +0,0 @@
--- broker_schedule table
--- Stores scheduled jobs (cron-like functionality)
-
-CREATE TABLE IF NOT EXISTS broker_schedule (
-    id_broker_schedule BIGSERIAL PRIMARY KEY,
-    name VARCHAR(255) NOT NULL UNIQUE,
-    cron_expr VARCHAR(100) NOT NULL,
-    enabled BOOLEAN NOT NULL DEFAULT true,
-    job_name VARCHAR(255) NOT NULL,
-    job_priority INTEGER NOT NULL DEFAULT 0,
-    job_queue INTEGER NOT NULL DEFAULT 1,
-    job_language VARCHAR(50) NOT NULL DEFAULT 'sql',
-    execute_str TEXT NOT NULL,
-    run_as VARCHAR(100),
-    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
-    updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
-    last_run_at TIMESTAMP WITH TIME ZONE,
-    next_run_at TIMESTAMP WITH TIME ZONE,
-
-    CONSTRAINT broker_schedule_job_queue_check CHECK (job_queue > 0)
-);
-
--- Indexes
-CREATE INDEX IF NOT EXISTS idx_broker_schedule_enabled ON broker_schedule(enabled);
-CREATE INDEX IF NOT EXISTS idx_broker_schedule_next_run ON broker_schedule(next_run_at) WHERE enabled = true;
-CREATE INDEX IF NOT EXISTS idx_broker_schedule_name ON broker_schedule(name);
-
--- Comments
-COMMENT ON TABLE broker_schedule IS 'Scheduled jobs (cron-like functionality)';
-COMMENT ON COLUMN broker_schedule.name IS 'Unique name for the schedule';
-COMMENT ON COLUMN broker_schedule.cron_expr IS 'Cron expression for scheduling';
-COMMENT ON COLUMN broker_schedule.enabled IS 'Whether the schedule is active';
-COMMENT ON COLUMN broker_schedule.job_name IS 'Name of the job to create';
-COMMENT ON COLUMN broker_schedule.execute_str IS 'SQL or code to execute';
-COMMENT ON COLUMN broker_schedule.last_run_at IS 'Last time the job was executed';
-COMMENT ON COLUMN broker_schedule.next_run_at IS 'Next scheduled execution time';
-
--- Trigger to update updated_at
-CREATE OR REPLACE FUNCTION tf_broker_schedule_update_timestamp()
-RETURNS TRIGGER AS $$
-BEGIN
-    NEW.updated_at = NOW();
-    RETURN NEW;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE TRIGGER t_broker_schedule_updated_at
-    BEFORE UPDATE ON broker_schedule
-    FOR EACH ROW
-    EXECUTE FUNCTION tf_broker_schedule_update_timestamp();

pkg/broker/install/sql/tables/03_broker_jobs.sql

@@ -1,62 +0,0 @@
--- broker_jobs table
--- Stores jobs to be executed by the broker
-
-CREATE TABLE IF NOT EXISTS broker_jobs (
-    id_broker_jobs BIGSERIAL PRIMARY KEY,
-    job_name VARCHAR(255) NOT NULL,
-    job_priority INTEGER NOT NULL DEFAULT 0,
-    job_queue INTEGER NOT NULL DEFAULT 1,
-    job_language VARCHAR(50) NOT NULL DEFAULT 'sql',
-    execute_str TEXT NOT NULL,
-    execute_result TEXT,
-    error_msg TEXT,
-    complete_status INTEGER NOT NULL DEFAULT 0,
-    run_as VARCHAR(100),
-    rid_broker_schedule BIGINT,
-    rid_broker_queueinstance BIGINT,
-    depends_on TEXT[],
-    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
-    updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
-    started_at TIMESTAMP WITH TIME ZONE,
-    completed_at TIMESTAMP WITH TIME ZONE,
-
-    CONSTRAINT broker_jobs_complete_status_check CHECK (complete_status IN (0, 1, 2, 3, 4)),
-    CONSTRAINT broker_jobs_job_queue_check CHECK (job_queue > 0),
-    CONSTRAINT fk_schedule FOREIGN KEY (rid_broker_schedule) REFERENCES broker_schedule(id_broker_schedule) ON DELETE SET NULL,
-    CONSTRAINT fk_instance FOREIGN KEY (rid_broker_queueinstance) REFERENCES broker_queueinstance(id_broker_queueinstance) ON DELETE SET NULL
-);
-
--- Indexes
-CREATE INDEX IF NOT EXISTS idx_broker_jobs_status ON broker_jobs(complete_status);
-CREATE INDEX IF NOT EXISTS idx_broker_jobs_queue ON broker_jobs(job_queue, complete_status, job_priority);
-CREATE INDEX IF NOT EXISTS idx_broker_jobs_schedule ON broker_jobs(rid_broker_schedule);
-CREATE INDEX IF NOT EXISTS idx_broker_jobs_instance ON broker_jobs(rid_broker_queueinstance);
-CREATE INDEX IF NOT EXISTS idx_broker_jobs_created ON broker_jobs(created_at);
-CREATE INDEX IF NOT EXISTS idx_broker_jobs_name ON broker_jobs(job_name, complete_status);
-
--- Comments
-COMMENT ON TABLE broker_jobs IS 'Job queue for broker execution';
-COMMENT ON COLUMN broker_jobs.job_name IS 'Name/description of the job';
-COMMENT ON COLUMN broker_jobs.job_priority IS 'Job priority (higher = more important)';
-COMMENT ON COLUMN broker_jobs.job_queue IS 'Queue number (allows parallel processing)';
-COMMENT ON COLUMN broker_jobs.job_language IS 'Execution language (sql, plpgsql, etc.)';
-COMMENT ON COLUMN broker_jobs.execute_str IS 'SQL or code to execute';
-COMMENT ON COLUMN broker_jobs.complete_status IS '0=pending, 1=running, 2=completed, 3=failed, 4=cancelled';
-COMMENT ON COLUMN broker_jobs.run_as IS 'User context to run the job as';
-COMMENT ON COLUMN broker_jobs.rid_broker_schedule IS 'Reference to schedule if job was scheduled';
-COMMENT ON COLUMN broker_jobs.rid_broker_queueinstance IS 'Instance that processed this job';
-COMMENT ON COLUMN broker_jobs.depends_on IS 'Array of job names that must be completed before this job can run';
-
--- Trigger to update updated_at
-CREATE OR REPLACE FUNCTION tf_broker_jobs_update_timestamp()
-RETURNS TRIGGER AS $$
-BEGIN
-    NEW.updated_at = NOW();
-    RETURN NEW;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE TRIGGER t_broker_jobs_updated_at
-    BEFORE UPDATE ON broker_jobs
-    FOR EACH ROW
-    EXECUTE FUNCTION tf_broker_jobs_update_timestamp();

pkg/broker/metrics/metrics.go

@@ -0,0 +1,130 @@
+// Package metrics defines the broker's Prometheus collectors and a small
+// embedded HTTP server that exposes them, both as the standard /metrics
+// exposition endpoint and as a single-page HTML dashboard.
+package metrics
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metrics holds every Prometheus collector the broker exposes. It is safe
+// for concurrent use, and a nil *Metrics is safe to call methods on (all
+// recording methods become no-ops), so callers that construct a Broker
+// without metrics enabled don't need to special-case it.
+type Metrics struct {
+	Registry *prometheus.Registry
+
+	jobsCompleted *prometheus.CounterVec
+	jobsFailed    *prometheus.CounterVec
+	jobsRequeued  *prometheus.CounterVec
+	jobDuration   *prometheus.HistogramVec
+	jobsQueued    *prometheus.GaugeVec
+	databaseCount prometheus.Gauge
+	queueCount    *prometheus.GaugeVec
+}
+
+// New creates a Metrics instance with a fresh (non-global) registry, so
+// multiple brokers can coexist in the same process -- e.g. in tests --
+// without colliding on prometheus.DefaultRegisterer.
+func New() *Metrics {
+	registry := prometheus.NewRegistry()
+
+	m := &Metrics{
+		Registry: registry,
+		jobsCompleted: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "broker_jobs_completed_total",
+			Help: "Total number of jobs that completed successfully.",
+		}, []string{"database", "job_group", "job_name"}),
+		jobsFailed: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "broker_jobs_failed_total",
+			Help: "Total number of jobs that were dead-lettered after exhausting retries.",
+		}, []string{"database", "job_group", "job_name"}),
+		jobsRequeued: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "broker_jobs_requeued_total",
+			Help: "Total number of job attempts that failed and were requeued for retry.",
+		}, []string{"database", "job_group", "job_name"}),
+		jobDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "broker_job_duration_seconds",
+			Help:    "Job execution duration in seconds, by group and name.",
+			Buckets: prometheus.DefBuckets,
+		}, []string{"database", "job_group", "job_name"}),
+		jobsQueued: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "broker_jobs_queued",
+			Help: "Current number of pending (not yet claimed) jobs, by database and queue.",
+		}, []string{"database", "queue"}),
+		databaseCount: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "broker_databases",
+			Help: "Number of database instances managed by this broker process.",
+		}),
+		queueCount: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "broker_queues",
+			Help: "Number of queues configured for a database instance.",
+		}, []string{"database"}),
+	}
+
+	registry.MustRegister(
+		m.jobsCompleted,
+		m.jobsFailed,
+		m.jobsRequeued,
+		m.jobDuration,
+		m.jobsQueued,
+		m.databaseCount,
+		m.queueCount,
+	)
+
+	return m
+}
+
+// RecordJobCompleted records a successfully completed job attempt.
+func (m *Metrics) RecordJobCompleted(database, group, name string, duration time.Duration) {
+	if m == nil {
+		return
+	}
+	m.jobsCompleted.WithLabelValues(database, group, name).Inc()
+	m.jobDuration.WithLabelValues(database, group, name).Observe(duration.Seconds())
+}
+
+// RecordJobFailed records a job attempt that was dead-lettered (attempts exhausted).
+func (m *Metrics) RecordJobFailed(database, group, name string, duration time.Duration) {
+	if m == nil {
+		return
+	}
+	m.jobsFailed.WithLabelValues(database, group, name).Inc()
+	m.jobDuration.WithLabelValues(database, group, name).Observe(duration.Seconds())
+}
+
+// RecordJobRequeued records a job attempt that failed but was requeued for retry.
+func (m *Metrics) RecordJobRequeued(database, group, name string, duration time.Duration) {
+	if m == nil {
+		return
+	}
+	m.jobsRequeued.WithLabelValues(database, group, name).Inc()
+	m.jobDuration.WithLabelValues(database, group, name).Observe(duration.Seconds())
+}
+
+// SetJobsQueued sets the current pending job count for a database/queue pair.
+func (m *Metrics) SetJobsQueued(database string, queue int, count float64) {
+	if m == nil {
+		return
+	}
+	m.jobsQueued.WithLabelValues(database, strconv.Itoa(queue)).Set(count)
+}
+
+// SetDatabaseCount sets the number of database instances managed by this process.
+func (m *Metrics) SetDatabaseCount(n int) {
+	if m == nil {
+		return
+	}
+	m.databaseCount.Set(float64(n))
+}
+
+// SetQueueCount sets the number of queues configured for a database instance.
+func (m *Metrics) SetQueueCount(database string, n int) {
+	if m == nil {
+		return
+	}
+	m.queueCount.WithLabelValues(database).Set(float64(n))
+}

pkg/broker/metrics/page.html

@@ -0,0 +1,215 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>pgsql-broker metrics</title>
+<style>
+  :root {
+    color-scheme: dark;
+    --bg: #0f1115;
+    --panel: #161a22;
+    --border: #262b36;
+    --text: #e6e9ef;
+    --muted: #8b93a7;
+    --accent: #5aa8ff;
+    --good: #4caf7d;
+    --bad: #e5636b;
+  }
+  * { box-sizing: border-box; }
+  body {
+    margin: 0;
+    padding: 2rem;
+    background: var(--bg);
+    color: var(--text);
+    font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
+  }
+  h1 {
+    font-size: 1.25rem;
+    margin: 0 0 0.25rem;
+  }
+  #status {
+    color: var(--muted);
+    font-size: 0.8rem;
+    margin-bottom: 1.5rem;
+  }
+  .grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(360px, 1fr));
+    gap: 1rem;
+  }
+  .card {
+    background: var(--panel);
+    border: 1px solid var(--border);
+    border-radius: 8px;
+    padding: 1rem;
+  }
+  .card h2 {
+    font-size: 0.9rem;
+    margin: 0 0 0.75rem;
+    color: var(--accent);
+    font-weight: 600;
+  }
+  .card .help {
+    color: var(--muted);
+    font-size: 0.75rem;
+    margin: -0.5rem 0 0.75rem;
+  }
+  table {
+    width: 100%;
+    border-collapse: collapse;
+    font-size: 0.8rem;
+  }
+  td, th {
+    text-align: left;
+    padding: 0.2rem 0.4rem 0.2rem 0;
+    border-bottom: 1px solid var(--border);
+  }
+  td.value {
+    text-align: right;
+    font-variant-numeric: tabular-nums;
+    white-space: nowrap;
+  }
+  td.labels {
+    color: var(--muted);
+  }
+  .completed .value { color: var(--good); }
+  .failed .value { color: var(--bad); }
+  .empty {
+    color: var(--muted);
+    font-size: 0.8rem;
+  }
+</style>
+</head>
+<body>
+  <h1>pgsql-broker metrics</h1>
+  <div id="status">loading&hellip;</div>
+  <div class="grid" id="grid"></div>
+
+<script>
+(function () {
+  "use strict";
+
+  // Metric name -> { help, className } for the cards we render, in order.
+  var METRICS = [
+    { name: "broker_jobs_completed_total", title: "Jobs completed", cls: "completed" },
+    { name: "broker_jobs_failed_total", title: "Jobs failed (dead-lettered)", cls: "failed" },
+    { name: "broker_jobs_requeued_total", title: "Jobs requeued (retries)", cls: "" },
+    { name: "broker_jobs_queued", title: "Jobs currently queued", cls: "" },
+    { name: "broker_job_duration_seconds_sum", title: "Job duration, total seconds by group/name", cls: "" },
+    { name: "broker_job_duration_seconds_count", title: "Job duration, sample count by group/name", cls: "" },
+    { name: "broker_databases", title: "Databases managed", cls: "" },
+    { name: "broker_queues", title: "Queues per database", cls: "" }
+  ];
+
+  // Parses Prometheus text exposition format into { name: [{labels, value}] }.
+  function parseMetrics(text) {
+    var byName = {};
+    var lines = text.split("\n");
+    for (var i = 0; i < lines.length; i++) {
+      var line = lines[i];
+      if (!line || line[0] === "#") continue;
+
+      var name, labels = {}, rest;
+      var braceIdx = line.indexOf("{");
+      var spaceIdx;
+      if (braceIdx !== -1) {
+        name = line.slice(0, braceIdx);
+        var closeIdx = line.indexOf("}", braceIdx);
+        if (closeIdx === -1) continue;
+        var labelStr = line.slice(braceIdx + 1, closeIdx);
+        var labelRe = /(\w+)="((?:[^"\\]|\\.)*)"/g;
+        var m;
+        while ((m = labelRe.exec(labelStr)) !== null) {
+          labels[m[1]] = m[2].replace(/\\"/g, '"').replace(/\\\\/g, "\\");
+        }
+        rest = line.slice(closeIdx + 1).trim();
+      } else {
+        spaceIdx = line.indexOf(" ");
+        if (spaceIdx === -1) continue;
+        name = line.slice(0, spaceIdx);
+        rest = line.slice(spaceIdx + 1).trim();
+      }
+
+      var value = parseFloat(rest.split(" ")[0]);
+      if (isNaN(value)) continue;
+
+      if (!byName[name]) byName[name] = [];
+      byName[name].push({ labels: labels, value: value });
+    }
+    return byName;
+  }
+
+  function formatLabels(labels) {
+    var keys = Object.keys(labels).sort();
+    return keys.map(function (k) { return k + "=" + labels[k]; }).join(", ");
+  }
+
+  function formatValue(v) {
+    if (Number.isInteger(v)) return String(v);
+    return v.toFixed(3);
+  }
+
+  function render(byName) {
+    var grid = document.getElementById("grid");
+    grid.innerHTML = "";
+
+    METRICS.forEach(function (spec) {
+      var series = byName[spec.name] || [];
+      var card = document.createElement("div");
+      card.className = "card " + spec.cls;
+
+      var h2 = document.createElement("h2");
+      h2.textContent = spec.title;
+      card.appendChild(h2);
+
+      if (series.length === 0) {
+        var empty = document.createElement("div");
+        empty.className = "empty";
+        empty.textContent = "no data yet";
+        card.appendChild(empty);
+      } else {
+        var table = document.createElement("table");
+        series.sort(function (a, b) { return b.value - a.value; });
+        series.forEach(function (s) {
+          var tr = document.createElement("tr");
+          var tdLabels = document.createElement("td");
+          tdLabels.className = "labels";
+          tdLabels.textContent = formatLabels(s.labels) || "(none)";
+          var tdValue = document.createElement("td");
+          tdValue.className = "value";
+          tdValue.textContent = formatValue(s.value);
+          tr.appendChild(tdLabels);
+          tr.appendChild(tdValue);
+          table.appendChild(tr);
+        });
+        card.appendChild(table);
+      }
+
+      grid.appendChild(card);
+    });
+  }
+
+  function refresh() {
+    fetch("metrics", { cache: "no-store" })
+      .then(function (resp) {
+        if (!resp.ok) throw new Error("HTTP " + resp.status);
+        return resp.text();
+      })
+      .then(function (text) {
+        render(parseMetrics(text));
+        document.getElementById("status").textContent =
+          "last updated " + new Date().toLocaleTimeString();
+      })
+      .catch(function (err) {
+        document.getElementById("status").textContent =
+          "failed to load metrics: " + err.message;
+      });
+  }
+
+  refresh();
+  setInterval(refresh, 5000);
+})();
+</script>
+</body>
+</html>

pkg/broker/metrics/server.go

@@ -0,0 +1,74 @@
+package metrics
+
+import (
+	"context"
+	_ "embed"
+	"fmt"
+	"net"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
+)
+
+//go:embed page.html
+var dashboardHTML []byte
+
+// Server is the embedded HTTP server exposing /metrics (standard Prometheus
+// exposition format) and / (a single-page HTML dashboard that polls
+// /metrics).
+type Server struct {
+	httpServer *http.Server
+	listener   net.Listener
+	logger     adapter.Logger
+}
+
+// NewServer builds a Server bound to addr (e.g. "127.0.0.1:9469"). Binding
+// happens immediately so a port conflict is reported to the caller rather
+// than surfacing later in a background goroutine.
+func NewServer(m *Metrics, addr string, logger adapter.Logger) (*Server, error) {
+	listener, err := net.Listen("tcp", addr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to bind metrics server to %s: %w", addr, err)
+	}
+
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.HandlerFor(m.Registry, promhttp.HandlerOpts{}))
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/" {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write(dashboardHTML)
+	})
+
+	return &Server{
+		httpServer: &http.Server{Handler: mux},
+		listener:   listener,
+		logger:     logger.With("component", "metrics-server"),
+	}, nil
+}
+
+// Addr returns the actual bound address (useful when addr was given with a
+// ":0" port).
+func (s *Server) Addr() string {
+	return s.listener.Addr().String()
+}
+
+// Start serves in the background. It returns immediately; Serve errors
+// (other than a clean Shutdown) are logged.
+func (s *Server) Start() {
+	s.logger.Info("metrics server listening", "addr", s.Addr())
+	go func() {
+		if err := s.httpServer.Serve(s.listener); err != nil && err != http.ErrServerClosed {
+			s.logger.Error("metrics server stopped unexpectedly", "error", err)
+		}
+	}()
+}
+
+// Stop gracefully shuts down the server.
+func (s *Server) Stop(ctx context.Context) error {
+	return s.httpServer.Shutdown(ctx)
+}

pkg/broker/models/models.go

@@ -16,10 +16,24 @@ type Job struct {
 	RunAs          string    `json:"run_as"`
 	UserLogin      string    `json:"user_login"`
 	ScheduleID     int64     `json:"schedule_id"`
+	TenantID       string    `json:"tenant_id"`
+	AttemptCount   int       `json:"attempt_count"`
+	MaxAttempts    int       `json:"max_attempts"`
+	LeaseToken     string    `json:"lease_token,omitempty"`
+	IdempotencyKey string    `json:"idempotency_key,omitempty"`
 	CreatedAt      time.Time `json:"created_at"`
 	UpdatedAt      time.Time `json:"updated_at"`
 }
 
+// WakeNotification is the payload sent over pg_notify('broker.event', ...).
+// It carries only what a worker needs to decide whether to wake: the queue
+// number. job_id is included solely for logging -- workers always re-claim
+// via broker_get rather than executing the notified id directly.
+type WakeNotification struct {
+	Queue int   `json:"queue"`
+	JobID int64 `json:"job_id,omitempty"`
+}
+
 // Instance represents a broker instance
 type Instance struct {
 	ID          int64     `json:"id"`

pkg/broker/queue/queue.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
-	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/models"
+	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/metrics"
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/worker"
 )
 
@@ -25,14 +25,18 @@ type Queue struct {
 
 // Config holds queue configuration
 type Config struct {
-	Number            int
+	Number       int
-	InstanceID        int64
+	InstanceID   int64
-	WorkerCount       int
+	WorkerCount  int
-	DBAdapter         adapter.DBAdapter
+	DBAdapter    adapter.DBAdapter
-	Logger            adapter.Logger
+	Logger       adapter.Logger
-	BufferSize        int
+	BufferSize   int
-	TimerSeconds      int
+	TimerSeconds int
-	FetchSize         int
+	FetchSize    int
+	TenantID     string
+	LeaseSeconds int
+	Metrics      *metrics.Metrics
+	DatabaseName string
 }
 
 // New creates a new queue manager
@@ -70,6 +74,10 @@ func (q *Queue) Start(cfg Config) error {
 			BufferSize:   cfg.BufferSize,
 			TimerSeconds: cfg.TimerSeconds,
 			FetchSize:    cfg.FetchSize,
+			TenantID:     cfg.TenantID,
+			LeaseSeconds: cfg.LeaseSeconds,
+			Metrics:      cfg.Metrics,
+			DatabaseName: cfg.DatabaseName,
 		})
 
 		if err := w.Start(q.ctx); err != nil {
@@ -109,24 +117,16 @@ func (q *Queue) Stop() error {
 	return nil
 }
 
-// AddJob adds a job to the least busy worker
+// Wake signals every worker in the queue to check for available jobs
-func (q *Queue) AddJob(job models.Job) error {
+// immediately. There is no job hand-off: fetching is always DB-driven via
+// broker_get, so waking a worker that finds nothing is harmless.
+func (q *Queue) Wake() {
 	q.mu.RLock()
 	defer q.mu.RUnlock()
 
-	if len(q.workers) == 0 {
-		return fmt.Errorf("no workers available")
-	}
-
-	// Simple round-robin: use first available worker
-	// Could be enhanced with load balancing
 	for _, w := range q.workers {
-		if err := w.AddJob(job); err == nil {
+		w.Wake()
-			return nil
-		}
 	}
-
-	return fmt.Errorf("all workers are busy")
 }
 
 // GetStats returns statistics for all workers in the queue

pkg/broker/worker/worker.go

@@ -2,13 +2,14 @@ package worker
 
 import (
 	"context"
-	"database/sql" // Import sql package
+	"database/sql"
 	"fmt"
+	"runtime/debug"
 	"sync"
 	"time"
 
 	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
-	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/models"
+	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/metrics"
 )
 
 // Worker represents a single job processing worker
@@ -18,7 +19,7 @@ type Worker struct {
 	InstanceID   int64
 	db           adapter.DBAdapter
 	logger       adapter.Logger
-	jobChan      chan models.Job
+	wakeChan     chan struct{}
 	shutdown     chan struct{}
 	wg           *sync.WaitGroup
 	running      bool
@@ -27,6 +28,10 @@ type Worker struct {
 	jobsHandled  int64
 	timerSeconds int
 	fetchSize    int
+	tenantID     string
+	leaseSeconds int
+	metrics      *metrics.Metrics
+	databaseName string
 }
 
 // Stats holds worker statistics
@@ -46,21 +51,34 @@ type Config struct {
 	BufferSize   int
 	TimerSeconds int
 	FetchSize    int
+	TenantID     string
+	LeaseSeconds int
+	Metrics      *metrics.Metrics
+	DatabaseName string
 }
 
 // New creates a new worker
 func New(cfg Config) *Worker {
+	leaseSeconds := cfg.LeaseSeconds
+	if leaseSeconds <= 0 {
+		leaseSeconds = 60
+	}
+
 	return &Worker{
 		ID:           cfg.ID,
 		QueueNumber:  cfg.QueueNumber,
 		InstanceID:   cfg.InstanceID,
 		db:           cfg.DBAdapter,
 		logger:       cfg.Logger.With("worker_id", cfg.ID).With("queue", cfg.QueueNumber),
-		jobChan:      make(chan models.Job, cfg.BufferSize),
+		wakeChan:     make(chan struct{}, 1),
 		shutdown:     make(chan struct{}),
 		wg:           &sync.WaitGroup{},
 		timerSeconds: cfg.TimerSeconds,
 		fetchSize:    cfg.FetchSize,
+		tenantID:     cfg.TenantID,
+		leaseSeconds: leaseSeconds,
+		metrics:      cfg.Metrics,
+		databaseName: cfg.DatabaseName,
 	}
 }
 
@@ -77,11 +95,48 @@ func (w *Worker) Start(ctx context.Context) error {
 	w.logger.Info("worker starting")
 
 	w.wg.Add(1)
-	go w.processLoop(ctx)
+	go w.superviseProcessLoop(ctx)
 
 	return nil
 }
 
+// superviseProcessLoop runs processLoop for the life of the worker,
+// restarting it (after a short backoff) if it ever panics, so a bug in job
+// processing can never permanently kill this worker's goroutine.
+func (w *Worker) superviseProcessLoop(ctx context.Context) {
+	defer w.wg.Done()
+
+	for {
+		select {
+		case <-w.shutdown:
+			return
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		if w.runProcessLoopOnce(ctx) {
+			return
+		}
+
+		time.Sleep(time.Second)
+	}
+}
+
+// runProcessLoopOnce runs processLoop, recovering any panic. It returns true
+// if processLoop returned normally (shutdown/context done, no restart
+// needed) and false if it panicked (caller should restart it).
+func (w *Worker) runProcessLoopOnce(ctx context.Context) (clean bool) {
+	defer func() {
+		if r := recover(); r != nil {
+			w.logger.Error("worker panic recovered, restarting", "panic", r, "stack", string(debug.Stack()))
+			clean = false
+		}
+	}()
+	w.processLoop(ctx)
+	return true
+}
+
 // Stop gracefully stops the worker
 func (w *Worker) Stop() error {
 	w.mu.Lock()
@@ -103,35 +158,33 @@ func (w *Worker) Stop() error {
 	return nil
 }
 
-// AddJob adds a job to the worker's queue
+// Wake signals the worker to check for available jobs immediately, instead
-func (w *Worker) AddJob(job models.Job) error {
+// of waiting for the next timer tick. Fetching is always DB-driven (via
+// broker_get with FOR UPDATE SKIP LOCKED), so a redundant or coalesced wake
+// is harmless.
+func (w *Worker) Wake() {
 	select {
-	case w.jobChan <- job:
+	case w.wakeChan <- struct{}{}:
-		return nil
 	default:
-		return fmt.Errorf("worker %d job channel is full", w.ID)
 	}
 }
 
 // processLoop is the main worker processing loop
 func (w *Worker) processLoop(ctx context.Context) {
-	defer w.wg.Done()
-	defer w.recoverPanic()
-
 	timer := time.NewTimer(time.Duration(w.timerSeconds) * time.Second)
 	defer timer.Stop()
 
 	for {
 		select {
-		case job := <-w.jobChan:
+		case <-w.wakeChan:
 			w.updateActivity()
-			w.processJobs(ctx, &job)
+			w.processJobs(ctx)
 
 		case <-timer.C:
 			// Timer expired - fetch jobs from database
 			if w.timerSeconds > 0 {
 				w.updateActivity()
-				w.processJobs(ctx, nil)
+				w.processJobs(ctx)
 			}
 			timer.Reset(time.Duration(w.timerSeconds) * time.Second)
 
@@ -147,95 +200,161 @@ func (w *Worker) processLoop(ctx context.Context) {
 }
 
 // processJobs processes jobs from the queue within a transaction
-func (w *Worker) processJobs(ctx context.Context, specificJob *models.Job) {
+func (w *Worker) processJobs(ctx context.Context) {
 	defer w.recoverPanic()
 
 	for i := 0; i < w.fetchSize; i++ {
-
+		tx, err := w.db.Begin(ctx)
-		tx, err := w.db.Begin(ctx) // Start transaction
 		if err != nil {
 			w.logger.Error("failed to begin transaction", "error", err)
 			return
 		}
 
-		var jobID int64
+		if err := w.setTenantTx(ctx, tx); err != nil {
-
+			if rbErr := tx.Rollback(); rbErr != nil {
-		if specificJob != nil && specificJob.ID > 0 {
+				w.logger.Error("failed to rollback transaction", "error", rbErr)
-			jobID = specificJob.ID
-			specificJob = nil // Only process once
-		} else {
-			jobID, err = w.fetchNextJobTx(ctx, tx) // Use transaction
-			if err != nil {
-				tx.Rollback() // Rollback on fetch error
-				w.logger.Error("failed to fetch job", "error", err)
-				return
 			}
+			w.logger.Error("failed to set tenant", "error", err)
+			return
+		}
+
+		jobID, leaseToken, err := w.fetchNextJobTx(ctx, tx)
+		if err != nil {
+			if rbErr := tx.Rollback(); rbErr != nil {
+				w.logger.Error("failed to rollback transaction", "error", rbErr)
+			}
+			w.logger.Error("failed to fetch job", "error", err)
+			return
 		}
 
 		if jobID <= 0 {
-			tx.Rollback() // No job found, rollback
+			// No job found, rollback
+			if rbErr := tx.Rollback(); rbErr != nil {
+				w.logger.Error("failed to rollback transaction", "error", rbErr)
+			}
 			return // No more jobs
 		}
 
+		jobName, jobGroup, err := w.fetchJobLabelsTx(ctx, tx, jobID)
+		if err != nil {
+			w.logger.Warn("failed to fetch job labels for metrics", "job_id", jobID, "error", err)
+		}
+
 		// Run the job
-		if err := w.runJobTx(ctx, tx, jobID); err != nil { // Use transaction
+		start := time.Now()
-			tx.Rollback() // Rollback on job execution error
+		jobStatus, err := w.runJobTx(ctx, tx, jobID, leaseToken)
+		duration := time.Since(start)
+		if err == nil {
+			w.recordJobMetric(jobStatus, jobGroup, jobName, duration)
+		}
+		if err != nil {
+			// Rollback on genuine infra failure
+			if rbErr := tx.Rollback(); rbErr != nil {
+				w.logger.Error("failed to rollback transaction", "error", rbErr)
+			}
 			w.logger.Error("failed to run job", "job_id", jobID, "error", err)
 		} else {
-			tx.Commit() // Commit if job successful
+			if err := tx.Commit(); err != nil {
+				w.logger.Error("failed to commit job", "job_id", jobID, "error", err)
+				continue
+			}
 			w.jobsHandled++
 		}
 	}
 }
 
-// fetchNextJobTx fetches the next job from the queue within a transaction
+// setTenantTx applies this worker's RLS tenant for the duration of tx.
-func (w *Worker) fetchNextJobTx(ctx context.Context, tx adapter.DBTransaction) (int64, error) {
+func (w *Worker) setTenantTx(ctx context.Context, tx adapter.DBTransaction) error {
+	tenantID := w.tenantID
+	if tenantID == "" {
+		tenantID = "default"
+	}
+	_, err := tx.Exec(ctx, "SELECT broker.broker_set_tenant($1)", tenantID)
+	return err
+}
+
+// fetchNextJobTx fetches the next job from the queue within a transaction,
+// claiming it with a lease that must be presented back to broker_run.
+func (w *Worker) fetchNextJobTx(ctx context.Context, tx adapter.DBTransaction) (jobID int64, leaseToken string, err error) {
 	var retval int
 	var errmsg string
 	var nullableJobID sql.NullInt64
+	var nullableLeaseToken sql.NullString
+
+	err = tx.QueryRow(ctx,
+		"SELECT p_retval, p_errmsg, p_job_id, p_lease_token FROM broker.broker_get($1, $2, $3)",
+		w.QueueNumber, w.InstanceID, w.leaseSeconds,
+	).Scan(&retval, &errmsg, &nullableJobID, &nullableLeaseToken)
+
+	if err != nil {
+		return 0, "", fmt.Errorf("query error: %w", err)
+	}
+
+	if retval > 0 {
+		return 0, "", fmt.Errorf("broker_get error: %s", errmsg)
+	}
+
+	if !nullableJobID.Valid {
+		return 0, "", nil
+	}
+
+	return nullableJobID.Int64, nullableLeaseToken.String, nil
+}
+
+// runJobTx executes a leased job within a transaction. It only returns an
+// error (triggering a rollback of the claim) on a genuine infra failure --
+// job outcomes reported via p_job_status (requeued/completed/dead-lettered)
+// are always committed.
+func (w *Worker) runJobTx(ctx context.Context, tx adapter.DBTransaction, jobID int64, leaseToken string) (int, error) {
+	w.logger.Debug("running job", "job_id", jobID)
+
+	var retval int
+	var errmsg string
+	var jobStatus int
 
 	err := tx.QueryRow(ctx,
-		"SELECT p_retval, p_errmsg, p_job_id FROM broker_get($1, $2)",
+		"SELECT p_retval, p_errmsg, p_job_status FROM broker.broker_run($1, $2)",
-		w.QueueNumber, w.InstanceID,
+		jobID, leaseToken,
-	).Scan(&retval, &errmsg, &nullableJobID)
+	).Scan(&retval, &errmsg, &jobStatus)
 
 	if err != nil {
 		return 0, fmt.Errorf("query error: %w", err)
 	}
 
 	if retval > 0 {
-		return 0, fmt.Errorf("broker_get error: %s", errmsg)
+		return 0, fmt.Errorf("broker_run error: %s", errmsg)
 	}
 
-	if !nullableJobID.Valid {
+	w.logger.Debug("job finished", "job_id", jobID, "job_status", jobStatus)
-		return 0, nil
+	return jobStatus, nil
-	}
-
-	return nullableJobID.Int64, nil
 }
 
-// runJobTx executes a job within a transaction
+// fetchJobLabelsTx looks up the job_name/job_group of jobID for metric
-func (w *Worker) runJobTx(ctx context.Context, tx adapter.DBTransaction, jobID int64) error {
+// labeling. Best-effort: callers log and continue on error rather than
-	w.logger.Debug("running job", "job_id", jobID)
+// failing the job over a metrics lookup.
-
+func (w *Worker) fetchJobLabelsTx(ctx context.Context, tx adapter.DBTransaction, jobID int64) (jobName, jobGroup string, err error) {
-	var retval int
+	err = tx.QueryRow(ctx,
-	var errmsg string
+		"SELECT job_name, job_group FROM broker.broker_jobs WHERE id_broker_jobs = $1",
-
-	err := tx.QueryRow(ctx,
-		"SELECT p_retval, p_errmsg FROM broker_run($1)",
 		jobID,
-	).Scan(&retval, &errmsg)
+	).Scan(&jobName, &jobGroup)
-
 	if err != nil {
-		return fmt.Errorf("query error: %w", err)
+		return "", "", fmt.Errorf("query error: %w", err)
 	}
+	return jobName, jobGroup, nil
+}
 
-	if retval > 0 {
+// recordJobMetric routes a finished job attempt to the appropriate
-		return fmt.Errorf("broker_run error: %s", errmsg)
+// Prometheus counter/histogram based on the p_job_status broker_run
+// reported (0=requeued, 2=completed, 3=dead-lettered).
+func (w *Worker) recordJobMetric(jobStatus int, jobGroup, jobName string, duration time.Duration) {
+	switch jobStatus {
+	case 2:
+		w.metrics.RecordJobCompleted(w.databaseName, jobGroup, jobName, duration)
+	case 3:
+		w.metrics.RecordJobFailed(w.databaseName, jobGroup, jobName, duration)
+	case 0:
+		w.metrics.RecordJobRequeued(w.databaseName, jobGroup, jobName, duration)
 	}
-
-	w.logger.Debug("job completed", "job_id", jobID)
-	return nil
 }
 
 // updateActivity updates the last activity timestamp
@@ -255,6 +374,6 @@ func (w *Worker) GetStats() (lastActivity time.Time, jobsHandled int64, running
 // recoverPanic recovers from panics in the worker
 func (w *Worker) recoverPanic() {
 	if r := recover(); r != nil {
-		w.logger.Error("worker panic recovered", "panic", r)
+		w.logger.Error("worker panic recovered", "panic", r, "stack", string(debug.Stack()))
 	}
 }

plan/docs/recommendations.md

@@ -1,8 +1,10 @@
 # PostgreSQL Schema and Function Recommendations
-
+/home/warkanum/.claude/plans/binary-puzzling-mountain.md
 **Review date:** 2026-09-14  
 **Scope:** `pkg/broker/install/sql` tables and functions, including their interaction with the Go worker.
 
+> Validated against authoritative PostgreSQL documentation on 2026-09-15; see [research_validation.md](./research_validation.md) for sources, verification method, and assumptions. Inline notes below marked "(validated: ...)" summarize that document's amendments.
+
 ## Conclusion
 
 The current schema is a credible internal prototype, but it is not yet a reliable production job queue. It has correctness problems that can strand jobs, lacks a safe RLS/execution-identity design, and needs standard operational queue features such as leases, retries, and migrations.
@@ -13,13 +15,13 @@ It is suitable only for a controlled environment with a single trusted operator
 
 | Priority | Problem | Recommendation |
 | --- | --- | --- |
-| Critical | A failed job is marked failed inside `broker_run`, then the function returns a nonzero result. The Go worker rolls back its transaction on nonzero results, so the failed update is rolled back and the job remains `running` forever. | Treat an expected job failure as a committed job outcome, while reserving nonzero function errors for infrastructure failures; alternatively persist the failure in a separate transaction. Add stale-running-job recovery. |
+| Critical | A failed job is marked failed inside `broker_run`, then the function returns a nonzero result. The Go worker rolls back its transaction on nonzero results, so the failed update is rolled back and the job remains `running` forever. | Treat an expected job failure as a committed job outcome, while reserving nonzero function errors for infrastructure failures; alternatively persist the failure in a separate transaction. Add stale-running-job recovery. (validated: converting `broker_run` to a `CREATE PROCEDURE` to commit independently only works if the outer `EXCEPTION WHEN OTHERS` block is removed/restructured — PL/pgSQL cannot run `COMMIT`/`ROLLBACK` inside an exception handler.) |
 | Critical | The notification handler passes a newly pending job directly to `broker_run`, but `broker_run` requires status `running`. The notification attempt fails and polling eventually processes the job. | Use NOTIFY only to wake workers. Every work attempt must claim a job through `broker_get` and `FOR UPDATE SKIP LOCKED`. |
 | Critical | Arbitrary queued SQL executes with the broker role. This cannot be made safe for multi-tenant/RLS use without an explicit execution model. | Prefer approved job procedures or job types with structured arguments. Add immutable tenant and execution-principal references. Use a least-privileged runtime role that does not own tenant tables and lacks `BYPASSRLS`. |
 | High | Active-instance registration is race-prone and the Go process can reuse another process's instance ID. | Fail a second startup. Use a session-held PostgreSQL advisory lock for exclusive ownership, with the instance table retained for observability and heartbeats. |
 | High | Jobs have no retry policy, lease expiry, dead-letter state, idempotency key, attempt counter, or backoff. | Add `attempt_count`, `max_attempts`, `available_at`, `leased_at`, `lease_expires_at`, `lease_token`, and terminal/dead-letter handling. |
 | High | Dependencies use mutable, non-unique names and treat running, failed, or cancelled dependencies as satisfied. | Replace `depends_on text[]` with `broker_job_dependency(job_id, depends_on_job_id)`. Permit execution only when all dependencies completed successfully, and define behaviour for failed dependencies. |
-| High | Schema installation is not idempotent because existing triggers cause reinstallation failures. There is no migration/version tracking. | Use ordered, transactional migrations with a schema-version table. Make trigger creation idempotent or recreate triggers safely. |
+| High | Schema installation is not idempotent because existing triggers cause reinstallation failures. There is no migration/version tracking. | Use ordered, transactional migrations with a schema-version table. Make trigger creation idempotent or recreate triggers safely. (validated: `CREATE OR REPLACE TRIGGER` requires PostgreSQL 14+; on earlier versions use `DROP TRIGGER IF EXISTS` followed by `CREATE TRIGGER`.) |
 | Medium | Job execution is inside the claim transaction. It is atomic but holds locks through arbitrary job SQL and prevents transaction-controlling work. | Explicitly choose a short lease/ack design with idempotent jobs, or document the atomic transaction limitation and restrict job types accordingly. |
 | Medium | `run_as` is unused. `broker_set` permits persistent session changes and its `search_path` branch is unsafe. | Remove identity controls until fully designed. Use only whitelisted transaction-local settings via `SET LOCAL` or `set_config(..., true)`. |
 | Medium | Existing indexes do not exactly serve queue claiming by queue, pending status, descending priority, and creation time. | Add and validate a partial claim index: `(job_queue, job_priority DESC, created_at, id_broker_jobs) WHERE complete_status = 0`. |
@@ -60,7 +62,7 @@ Use a dedicated `broker` schema and separate roles:
 2. A least-privileged broker runtime role, with no `BYPASSRLS` and no ownership of tenant tables.
 3. Narrowly scoped enqueue/application roles.
 
-RLS must use immutable `tenant_id` and a transaction-local, trusted tenant context. It must not be driven by arbitrary queued SQL or an unvalidated `run_as` field.
+RLS must use immutable `tenant_id` and a transaction-local, trusted tenant context. It must not be driven by arbitrary queued SQL or an unvalidated `run_as` field. If the runtime role ever owns the tenant tables (e.g. because it also ran migrations), it must also run `ALTER TABLE ... FORCE ROW LEVEL SECURITY`, since table owners otherwise bypass RLS by default ([Row Security Policies](https://www.postgresql.org/docs/current/ddl-rowsecurity.html)).
 
 ## Delivery order
 
@@ -161,7 +163,7 @@ The leader lock must protect only leader duties such as schedule creation and re
 ### PostgreSQL operating requirements
 
 1. Use a dedicated, session-persistent connection for `LISTEN`; transaction-pooling proxies cannot safely carry listener state. Use separate pooled connections for claims and execution.
-2. Treat `NOTIFY` as a low-latency wake-up only. Its payload is size-limited and notification delivery must not be the sole source of truth; periodic/bounded polling remains necessary.
+2. Treat `NOTIFY` as a low-latency wake-up only. Its payload must be shorter than 8000 bytes by default ([NOTIFY](https://www.postgresql.org/docs/current/sql-notify.html)) and notification delivery must not be the sole source of truth; periodic/bounded polling remains necessary.
 3. Tune connection-pool sizes per database capacity, rather than multiplying workers and connections without a budget.
 4. Monitor queue depth, oldest-ready-job age, claim latency, execution latency, lock waits, dead tuples, autovacuum progress, index size, WAL volume, replication lag, and database connection saturation.
 5. Load-test realistic payload sizes, tenant distributions, job durations, retry rates, and failure modes before selecting partitions, worker counts, or autovacuum settings.

plan/docs/research_validation.md

@@ -0,0 +1,90 @@
+# Research Validation: PostgreSQL Broker Recommendations
+
+**Review date:** 2026-09-15
+**Scope:** Independent validation of `plan/docs/recommendations.md` and `plan/docs/security_audit.md` (commit 602997b) against authoritative PostgreSQL documentation and the current source (`pkg/broker/install/sql/**`, `pkg/broker/worker/worker.go`, `pkg/broker/database_instance.go`). Performed for issue #1.
+
+This document does not repeat the recommendations; it records what was checked, what is a verified PostgreSQL fact vs. a design recommendation, exact sources, assumptions, and any amendments to the existing docs. All PostgreSQL doc links point at the "current" (development) manual as served by postgresql.org at review time; features cited are stable and present in all currently supported PostgreSQL major versions (13+) unless a version is noted.
+
+## Method
+
+Each claim below was checked one of two ways:
+- **Source-verified**: read the actual file/line in this repository.
+- **Doc-verified**: fetched the cited PostgreSQL manual page and quoted the operative sentence.
+
+Findings are marked accordingly. Anything not marked doc-verified or source-verified is this review's own recommendation/opinion, not an authoritative fact.
+
+## 1. Queue correctness / rollback-on-failure (Critical #1)
+
+**Source-verified.** `pkg/broker/install/sql/procedures/02_broker_run.sql` declares `broker_run` as `CREATE OR REPLACE FUNCTION ... LANGUAGE plpgsql`, not a procedure. On a job error it sets `complete_status = 3` (failed) via `UPDATE` and returns a nonzero `p_retval`. `pkg/broker/worker/worker.go:155-185` opens a transaction with `w.db.Begin(ctx)`, calls `broker_run` inside it, and calls `tx.Rollback()` whenever the returned error is non-nil (line 182), which undoes the `UPDATE ... complete_status = 3` from the same transaction. The job's `complete_status` therefore remains `1` (running) after rollback, exactly as the recommendation states.
+
+**Doc-verified root cause.** PostgreSQL functions (`CREATE FUNCTION`) cannot issue `COMMIT`/`ROLLBACK`; only procedures invoked via top-level (or uninterrupted) `CALL`/`DO` can, and even then not inside an `EXCEPTION` block (which starts a subtransaction) or a non-read-only cursor loop. Source: [PL/pgSQL Transaction Management](https://www.postgresql.org/docs/current/plpgsql-transactions.html). This confirms the recommendation's implicit assumption: converting `broker_run` to a `CREATE PROCEDURE` would let it commit the failure record independently of the caller's transaction outcome, but only if the exception is caught *outside* any `EXCEPTION` block that would otherwise need to commit — the current code's outer `EXCEPTION WHEN OTHERS` handler (lines 90-108) would need restructuring, since procedures cannot commit from within an exception handler either.
+
+**Amendment:** recommendations.md's fix ("treat failure as a committed outcome ... or persist the failure in a separate transaction") is correct but should explicitly note the procedure-conversion constraint above, since a naive `CREATE PROCEDURE` conversion that keeps the existing nested `EXCEPTION` block will not gain commit capability. Added as a caveat below in "Prioritized amendments."
+
+## 2. LISTEN/NOTIFY semantics (Critical #2, scaling §"PostgreSQL operating requirements")
+
+**Doc-verified**, from [NOTIFY](https://www.postgresql.org/docs/current/sql-notify.html):
+- Payload is capped: *"In the default configuration it must be shorter than 8000 bytes."* This is a specific, quotable number the existing docs describe only qualitatively ("size-limited"); worth stating explicitly.
+- Delivery is commit-gated: *"if a NOTIFY is executed inside a transaction, the notify events are not delivered until and unless the transaction is committed."* A listening session likewise only receives pending notifications *"just after the transaction is completed."* This supports the recommendation that NOTIFY must be treated as a best-effort wake-up, not a transactional guarantee — a crash between `COMMIT` and NOTIFY delivery, or a dropped/coalesced notification, is normal, expected PostgreSQL behavior, not a bug to work around.
+- Duplicate collapsing: *"If the same channel name is signaled multiple times with identical payload strings within the same transaction, only one instance ... is delivered."* This further supports "polling/claim scans remain the source of truth" in recommendations.md — even same-transaction NOTIFY calls are not reliably one-to-one with events.
+
+**Confirms recommendations.md's claim** that a transaction-pooling connection (e.g., PgBouncer in `transaction` or `statement` mode) cannot safely hold a `LISTEN` registration, since the physical server connection backing a pooled client connection can change between statements/transactions, silently dropping the `LISTEN` registration; PostgreSQL's own docs do not carry a proxy-specific warning, so this point in recommendations.md is a correct operational inference rather than a directly quotable PostgreSQL doc fact — labeled here as **recommendation, not doc-verified**, so the distinction is explicit for readers.
+
+**Amendment:** add the exact 8000-byte figure and the doc link to recommendations.md's NOTIFY section for precision.
+
+## 3. Claiming, `FOR UPDATE SKIP LOCKED` (Critical #2, High "leases/retries", scaling §"Claim and indexing strategy")
+
+**Source-verified.** `pkg/broker/install/sql/procedures/01_broker_get.sql:47-49` already uses `ORDER BY job_priority DESC, created_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED`, and claims + status transition happen in one function/statement scope — this part already matches the target model recommendations.md describes ("claim jobs atomically with FOR UPDATE SKIP LOCKED ... in one statement or a tightly scoped function").
+
+**Doc-verified**, from [SELECT ... FOR UPDATE/SHARE](https://www.postgresql.org/docs/current/sql-select.html#SQL-FOR-UPDATE-SHARE): *"Skipping locked rows provides an inconsistent view of the data, so this is not suitable for general purpose work, but can be used to avoid lock contention with multiple consumers accessing a queue-like table."* This is PostgreSQL's own documented, sanctioned use case for the exact pattern recommendations.md proposes — strong confirmation that `SKIP LOCKED` is the correct primitive, not just this project's convention.
+
+**Gap confirmed:** `broker_get`'s dependency check (lines 37-46) only excludes a candidate when a dependency's `complete_status = 0` (pending). A dependency in `running` (1) or `failed` (3) is not pending, so `NOT EXISTS (... dep.complete_status = 0)` is satisfied and the job is claimable — this is source-verified and matches the security_audit.md "High: dependency semantics are incorrect" finding exactly.
+
+## 4. Row-Level Security and execution identity (Critical #3)
+
+**Doc-verified**, from [Row Security Policies](https://www.postgresql.org/docs/current/ddl-rowsecurity.html): *"Superusers and roles with the BYPASSRLS attribute always bypass the row security system... Table owners normally bypass row security as well, though a table owner can choose to be subject to row security with ALTER TABLE ... FORCE ROW LEVEL SECURITY."* This directly confirms both audit documents' repeated point that a broker runtime role must not be a table owner and must not carry `BYPASSRLS`, and that `FORCE ROW LEVEL SECURITY` is required if the runtime role happens to own the tenant tables (e.g., because it ran the migrations) — this is a specific, actionable detail the recommendation summarizes correctly but doesn't cite by clause name.
+
+**Doc-verified**, policies are ordinary boolean expressions evaluated with the caller's privileges and can reference `current_setting(...)` (session/transaction-local GUCs), `current_user`, or connection metadata like `pg_catalog.inet_client_addr()`. This substantiates the recommendation to drive tenant context from *"a transaction-local, trusted tenant context"* — `set_config(name, value, is_local => true)` / `SET LOCAL` is the standard, documented mechanism for a transaction-scoped GUC a policy can read via `current_setting`, matching recommendations.md's `broker_set` guidance (use only "whitelisted transaction-local settings via SET LOCAL or set_config(..., true)"). Source: [Set Session Authorization / SET](https://www.postgresql.org/docs/current/sql-set.html) (`SET LOCAL` scoping) — reviewed for consistency; content matches long-standing, stable PostgreSQL behavior.
+
+**Assessment:** Both audit documents' RLS/identity recommendations are consistent with documented PostgreSQL behavior. No amendment needed beyond adding the `FORCE ROW LEVEL SECURITY` clause name for precision (see amendments).
+
+## 5. Single-instance ownership / advisory locks (High, security_audit.md)
+
+**Doc-verified**, from [Advisory Locks](https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS) and [Advisory Lock Functions](https://www.postgresql.org/docs/current/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS):
+- Session-level advisory locks (`pg_advisory_lock(key bigint)` / `pg_advisory_lock(key1 int, key2 int)`) persist across transactions within the same session and are *"automatically cleaned up by the server at the end of the session,"* including on ungraceful client/network disconnects — the server reliably releases them when the backend session ends. Non-blocking variants exist (`pg_try_advisory_lock`).
+- This confirms recommendations.md's proposal ("session-held PostgreSQL advisory lock for exclusive ownership") is a sound, standard pattern for exactly this exclusive-singleton problem: unlike the current row-count check in `broker_register_instance.sql`, an advisory lock held for the life of the broker's connection is automatically released by the server if the process dies or the socket drops, without needing a heartbeat-based takeover for the *lock* itself (a heartbeat is still useful for observability/monitoring, which recommendations.md keeps).
+
+**Assessment:** confirmed as documented, standard practice; no amendment needed.
+
+## 6. Dependencies, run groups, and scheduling
+
+These sections are primarily forward-looking design (normalized dependency tables, cron scheduling with IANA timezones, misfire policies) rather than claims about current PostgreSQL behavior, so there is little to doc-verify beyond generic SQL capabilities already confirmed above (indexed joins, `FOR UPDATE SKIP LOCKED` composability with an `EXISTS` dependency predicate, unique constraints for idempotent schedule materialization). One specific fact worth flagging:
+
+**Doc-verified**, from [Data Types — Date/Time Types](https://www.postgresql.org/docs/current/datatype-datetime.html) (spot-checked against the operative paragraphs — timezone conversion for `timestamptz` is well-established stable behavior): PostgreSQL stores `timestamptz` internally in UTC and converts to/from the session's `TimeZone` setting on display/input; it does not natively track "the schedule's own IANA zone" per row. This confirms recommendations.md's instruction to compute `next_run_at` in the schedule's configured zone *before* persisting as `timestamptz`, and to store the IANA zone name as its own column (not rely on the session `TimeZone` GUC) — the recommendation is correctly stated and necessary, since PostgreSQL will not do zone-aware scheduling arithmetic for you.
+
+No corrections needed to these sections; they are consistent with standard PostgreSQL capabilities and normalization practice.
+
+## 7. Migrations and idempotent installation
+
+**Source-verified.** `pkg/broker/install/sql/tables/02_broker_schedule.sql` and `03_broker_jobs.sql` use `CREATE TABLE IF NOT EXISTS`, but trigger creation in the same files uses plain `CREATE TRIGGER` (no `IF NOT EXISTS`/`OR REPLACE` guard). PostgreSQL's `CREATE TRIGGER` (see [CREATE TRIGGER](https://www.postgresql.org/docs/current/sql-createtrigger.html)) only gained `OR REPLACE` in PostgreSQL 14+; there is no `CREATE TRIGGER IF NOT EXISTS` in any version. This confirms the audit's claim that re-running install fails on existing triggers, and adds a version constraint the docs should carry: even the `CREATE OR REPLACE TRIGGER` fix requires PostgreSQL ≥ 14, otherwise the code must `DROP TRIGGER IF EXISTS ... ; CREATE TRIGGER ...` instead.
+
+**Amendment:** add this PG-14 minimum-version caveat to recommendations.md's "installer is not rerunnable" item.
+
+## 8. Scaling, maintenance, observability, operational safety
+
+These sections describe conventional, well-established PostgreSQL operational practice (partial indexes for hot queue predicates, time-based declarative partitioning for append-only history, bounded-batch deletes, autovacuum tuning, `EXPLAIN (ANALYZE, BUFFERS)` for validating index choice). Spot checks against [CREATE INDEX](https://www.postgresql.org/docs/current/sql-createindex.html) (partial indexes via `WHERE`) and [Table Partitioning](https://www.postgresql.org/docs/current/ddl-partitioning.html) (declarative range partitioning, detaching old partitions) confirm the described mechanisms exist and behave as summarized; no unsupported or version-inconsistent claims were found. No amendments needed.
+
+## Assumptions
+
+- Target deployment uses a currently-supported PostgreSQL major version (13–18); version-specific caveats above (e.g., `CREATE OR REPLACE TRIGGER` needing PG 14+) are called out explicitly rather than assumed away.
+- "Doc-verified" reflects the current/development manual at review time; behavior cited (NOTIFY commit-gating, SKIP LOCKED semantics, RLS bypass rules, advisory lock lifecycle, PL/pgSQL transaction-control restrictions) has been stable across PostgreSQL major versions for many years and is not expected to differ for older supported versions.
+- This review did not have network access to a live PostgreSQL instance in this environment to run `EXPLAIN` or reproduce the rollback bug end-to-end; the rollback and dependency-satisfaction defects were confirmed by static reading of the SQL/Go source (see source-verified notes above), not by execution.
+
+## Prioritized amendments to `plan/docs/recommendations.md`
+
+1. **Critical #1 (rollback bug fix):** note that converting `broker_run`/`broker_get` to `CREATE PROCEDURE` for independent commit only works if the outer `EXCEPTION WHEN OTHERS` block is removed or restructured, since procedures cannot run `COMMIT`/`ROLLBACK` inside an exception handler (PL/pgSQL forms a subtransaction there). Source: [PL/pgSQL Transaction Management](https://www.postgresql.org/docs/current/plpgsql-transactions.html).
+2. **NOTIFY payload limit:** state the concrete 8000-byte default limit and link [NOTIFY](https://www.postgresql.org/docs/current/sql-notify.html) instead of only "size-limited."
+3. **RLS/table ownership:** name `ALTER TABLE ... FORCE ROW LEVEL SECURITY` explicitly as the mechanism needed if the runtime role ever owns tenant tables, per [Row Security Policies](https://www.postgresql.org/docs/current/ddl-rowsecurity.html).
+4. **Installer idempotency:** add that `CREATE OR REPLACE TRIGGER` requires PostgreSQL 14+; on earlier versions use `DROP TRIGGER IF EXISTS` followed by `CREATE TRIGGER`.
+
+These amendments have been applied to `plan/docs/recommendations.md` as short inline notes with links to this document; the substantive prioritization and design recommendations in that file are otherwise confirmed and unchanged.

tests/integration/connection_test.go

@@ -2,6 +2,7 @@ package integration
 
 import (
 	"database/sql"
+	"fmt"
 	"testing"
 	"time"
 
@@ -10,7 +11,7 @@ import (
 )
 
 func TestConnection(t *testing.T) {
-	connStr := "user=user password=password dbname=broker_test port=5433 sslmode=disable"
+	connStr := fmt.Sprintf("user=user password=password dbname=broker_test host=%s port=%d sslmode=disable", testDBHost(), testDBPort())
 	var db *sql.DB
 	var err error
 

tests/integration/rls_test.go

@@ -0,0 +1,146 @@
+package integration
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"testing"
+
+	_ "github.com/lib/pq"
+	"github.com/stretchr/testify/require"
+)
+
+// TestRLSTenantIsolation verifies that a non-superuser role without
+// BYPASSRLS, connected via broker_set_tenant, only ever sees jobs and
+// dependency rows for its own tenant -- the core guarantee behind the
+// broker_jobs/broker_job_dependency FORCE ROW LEVEL SECURITY policies.
+func TestRLSTenantIsolation(t *testing.T) {
+	ctx := context.Background()
+	adminDB := setupStage5Schema(t)
+
+	// A restricted role, no BYPASSRLS, mirroring sql/roles/0001_roles.sql's
+	// broker_runtime (superuser test DB roles already bypass RLS entirely,
+	// so this test would be meaningless against the "user" role).
+	_, err := adminDB.Exec("DROP ROLE IF EXISTS test_broker_runtime")
+	require.NoError(t, err)
+	_, err = adminDB.Exec("CREATE ROLE test_broker_runtime LOGIN PASSWORD 'test-pass' NOSUPERUSER NOBYPASSRLS")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if _, err := adminDB.Exec("REASSIGN OWNED BY test_broker_runtime TO CURRENT_USER"); err != nil {
+			t.Logf("warning: failed to reassign objects owned by test_broker_runtime: %v", err)
+		}
+		if _, err := adminDB.Exec("DROP OWNED BY test_broker_runtime"); err != nil {
+			t.Logf("warning: failed to drop grants owned by test_broker_runtime: %v", err)
+		}
+		if _, err := adminDB.Exec("DROP ROLE IF EXISTS test_broker_runtime"); err != nil {
+			t.Logf("warning: failed to drop role test_broker_runtime: %v", err)
+		}
+	})
+
+	for _, stmt := range []string{
+		"GRANT USAGE ON SCHEMA broker TO test_broker_runtime",
+		"GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA broker TO test_broker_runtime",
+		"GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA broker TO test_broker_runtime",
+		"GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA broker TO test_broker_runtime",
+	} {
+		_, err = adminDB.Exec(stmt)
+		require.NoError(t, err)
+	}
+
+	runtimeDB, err := sql.Open("postgres",
+		fmt.Sprintf("user=test_broker_runtime password=test-pass dbname=broker_test host=%s port=%d sslmode=disable options='-c search_path=broker,public'", testDBHost(), testDBPort()))
+	require.NoError(t, err)
+	defer runtimeDB.Close()
+	require.NoError(t, runtimeDB.Ping())
+
+	// broker_set_tenant uses SET LOCAL semantics (set_config(..., true)), so
+	// it only takes effect for the remainder of the transaction it runs in.
+	// Callers must set the tenant and perform the tenant-scoped operation in
+	// the same explicit transaction (as worker.go's processJobs does) -- two
+	// separate autocommitted statements would each run in their own
+	// transaction and the tenant setting would not carry over.
+	addJobAsTenant := func(tenant, name string) int64 {
+		conn, err := runtimeDB.Conn(ctx)
+		require.NoError(t, err)
+		defer conn.Close()
+
+		tx, err := conn.BeginTx(ctx, nil)
+		require.NoError(t, err)
+
+		_, err = tx.ExecContext(ctx, "SELECT broker.broker_set_tenant($1)", tenant)
+		require.NoError(t, err)
+
+		var retval int
+		var errmsg string
+		var jobID int64
+		err = tx.QueryRowContext(ctx, `
+			SELECT p_retval, p_errmsg, p_job_id FROM broker.broker_add_job(
+				$1, 'SELECT 1', 1, 0, 'sql', NULL, NULL, NULL, NULL, 1
+			)`, name,
+		).Scan(&retval, &errmsg, &jobID)
+		require.NoError(t, err)
+		require.Equal(t, 0, retval, errmsg)
+
+		require.NoError(t, tx.Commit())
+		return jobID
+	}
+
+	tenantAJob := addJobAsTenant("tenant-a", "tenant-a-job")
+	tenantBJob := addJobAsTenant("tenant-b", "tenant-b-job")
+	require.NotEqual(t, tenantAJob, tenantBJob)
+
+	// As tenant-a, only tenant-a's job must be visible. All of tenant-a's
+	// checks (including the broker_get claim below) share one explicit
+	// transaction so the SET LOCAL tenant context stays in effect throughout.
+	connA, err := runtimeDB.Conn(ctx)
+	require.NoError(t, err)
+	defer connA.Close()
+	txA, err := connA.BeginTx(ctx, nil)
+	require.NoError(t, err)
+	defer txA.Rollback()
+
+	_, err = txA.ExecContext(ctx, "SELECT broker.broker_set_tenant($1)", "tenant-a")
+	require.NoError(t, err)
+
+	var visibleCount int
+	err = txA.QueryRowContext(ctx, "SELECT COUNT(*) FROM broker.broker_jobs WHERE id_broker_jobs IN ($1, $2)",
+		tenantAJob, tenantBJob).Scan(&visibleCount)
+	require.NoError(t, err)
+	require.Equal(t, 1, visibleCount, "tenant-a must see only its own job, not tenant-b's")
+
+	var visibleName string
+	err = txA.QueryRowContext(ctx, "SELECT job_name FROM broker.broker_jobs WHERE id_broker_jobs = $1", tenantAJob).Scan(&visibleName)
+	require.NoError(t, err)
+	require.Equal(t, "tenant-a-job", visibleName)
+
+	// As tenant-b, only tenant-b's job must be visible.
+	connB, err := runtimeDB.Conn(ctx)
+	require.NoError(t, err)
+	defer connB.Close()
+	txB, err := connB.BeginTx(ctx, nil)
+	require.NoError(t, err)
+	defer txB.Rollback()
+
+	_, err = txB.ExecContext(ctx, "SELECT broker.broker_set_tenant($1)", "tenant-b")
+	require.NoError(t, err)
+
+	err = txB.QueryRowContext(ctx, "SELECT COUNT(*) FROM broker.broker_jobs WHERE id_broker_jobs IN ($1, $2)",
+		tenantAJob, tenantBJob).Scan(&visibleCount)
+	require.NoError(t, err)
+	require.Equal(t, 1, visibleCount, "tenant-b must see only its own job, not tenant-a's")
+	require.NoError(t, txB.Commit())
+
+	// broker_get run under tenant-a's context must never be able to claim
+	// tenant-b's job.
+	var claimedID sql.NullInt64
+	var getRetval int
+	var getErrmsg string
+	err = txA.QueryRowContext(ctx,
+		"SELECT p_retval, p_errmsg, p_job_id FROM broker.broker_get($1, NULL, $2)", 1, 60,
+	).Scan(&getRetval, &getErrmsg, &claimedID)
+	require.NoError(t, err)
+	require.Equal(t, 0, getRetval, getErrmsg)
+	require.True(t, claimedID.Valid)
+	require.Equal(t, tenantAJob, claimedID.Int64, "tenant-a's broker_get must only ever claim tenant-a's own job")
+	require.NoError(t, txA.Commit())
+}

tests/integration/stage5_test.go

@@ -0,0 +1,344 @@
+package integration
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"log/slog"
+	"testing"
+	"time"
+
+	_ "github.com/lib/pq"
+	"github.com/stretchr/testify/require"
+
+	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/adapter"
+	"git.warky.dev/wdevs/pgsql-broker/pkg/broker/install"
+)
+
+func stage5ConnStr() string {
+	return fmt.Sprintf("user=user password=password dbname=broker_test host=%s port=%d sslmode=disable", testDBHost(), testDBPort())
+}
+
+func newStage5Adapter(logger adapter.Logger) *adapter.PostgresAdapter {
+	return adapter.NewPostgresAdapter(adapter.PostgresConfig{
+		Host: testDBHost(), Port: testDBPort(), Database: "broker_test",
+		User: "user", Password: "password", SSLMode: "disable",
+		MaxOpenConns: 10, MaxIdleConns: 2,
+		ConnMaxLifetime: 5 * time.Minute, ConnMaxIdleTime: 10 * time.Minute,
+	}, logger)
+}
+
+// setupStage5Schema drops and re-installs a clean broker schema, returning a
+// superuser *sql.DB for direct SQL against it.
+func setupStage5Schema(t *testing.T) *sql.DB {
+	t.Helper()
+
+	db, err := connectWithRetry(stage5ConnStr(), 10, 2*time.Second)
+	require.NoError(t, err)
+
+	cleanupSchema(t, db)
+
+	logger := adapter.NewSlogLogger(slog.LevelWarn)
+	dbAdapter := newStage5Adapter(logger)
+	require.NoError(t, dbAdapter.Connect(context.Background()))
+	defer dbAdapter.Close()
+
+	installer := install.New(dbAdapter, logger)
+	require.NoError(t, installer.ApplyMigrations(context.Background()))
+
+	t.Cleanup(func() { db.Close() })
+	return db
+}
+
+// TestRepeatMigrationRunIsNoOp verifies applying the migration set a second
+// time (with nothing pending) makes no changes and reports no error.
+func TestRepeatMigrationRunIsNoOp(t *testing.T) {
+	ctx := context.Background()
+	setupStage5Schema(t)
+
+	logger := adapter.NewSlogLogger(slog.LevelWarn)
+	dbAdapter := newStage5Adapter(logger)
+	require.NoError(t, dbAdapter.Connect(ctx))
+	defer dbAdapter.Close()
+
+	installer := install.New(dbAdapter, logger)
+
+	pending, err := installer.PendingMigrations(ctx)
+	require.NoError(t, err)
+	require.Empty(t, pending, "no migrations should be pending right after install")
+
+	require.NoError(t, installer.ApplyMigrations(ctx))
+	require.NoError(t, installer.VerifyInstallation(ctx))
+}
+
+// TestDuplicateInstanceStartFails verifies that broker_register_instance
+// rejects a second registration under the same instance name while the
+// first holds the advisory lock, and succeeds again once it's released.
+func TestDuplicateInstanceStartFails(t *testing.T) {
+	ctx := context.Background()
+	db := setupStage5Schema(t)
+
+	connA, err := db.Conn(ctx)
+	require.NoError(t, err)
+	defer connA.Close()
+
+	var retval int
+	var errmsg string
+	var instanceID sql.NullInt64
+
+	err = connA.QueryRowContext(ctx,
+		"SELECT p_retval, p_errmsg, p_instance_id FROM broker.broker_register_instance($1,$2,$3,$4,$5)",
+		"dup-test", "host-a", 111, "test", 2,
+	).Scan(&retval, &errmsg, &instanceID)
+	require.NoError(t, err)
+	require.Equal(t, 0, retval, "first registration should succeed: %s", errmsg)
+	require.True(t, instanceID.Valid)
+
+	// Second registration under the same name, on a different connection,
+	// must fail while connA still holds the advisory lock.
+	connB, err := db.Conn(ctx)
+	require.NoError(t, err)
+	defer connB.Close()
+
+	err = connB.QueryRowContext(ctx,
+		"SELECT p_retval, p_errmsg, p_instance_id FROM broker.broker_register_instance($1,$2,$3,$4,$5)",
+		"dup-test", "host-b", 222, "test", 2,
+	).Scan(&retval, &errmsg, &instanceID)
+	require.NoError(t, err)
+	require.Equal(t, 3, retval, "second registration must be rejected by the advisory lock")
+	require.False(t, instanceID.Valid)
+
+	// Release the lock (as registerInstance's caller would on shutdown) and
+	// confirm a fresh registration then succeeds.
+	_, err = connA.ExecContext(ctx, "SELECT pg_advisory_unlock(hashtextextended($1, 0))", "broker:dup-test")
+	require.NoError(t, err)
+	require.NoError(t, connA.Close())
+
+	connC, err := db.Conn(ctx)
+	require.NoError(t, err)
+	defer connC.Close()
+
+	err = connC.QueryRowContext(ctx,
+		"SELECT p_retval, p_errmsg, p_instance_id FROM broker.broker_register_instance($1,$2,$3,$4,$5)",
+		"dup-test", "host-c", 333, "test", 2,
+	).Scan(&retval, &errmsg, &instanceID)
+	require.NoError(t, err)
+	require.Equal(t, 0, retval, "registration should succeed again once the lock is released: %s", errmsg)
+}
+
+// TestJobDependencies covers dependency ordering (a job is not claimable
+// while an incomplete dependency exists), rejection of a direct
+// self-dependency at the table level, and idempotent duplicate-dependency
+// inserts.
+func TestJobDependencies(t *testing.T) {
+	ctx := context.Background()
+	db := setupStage5Schema(t)
+
+	addJob := func(name string, deps string) int64 {
+		var retval int
+		var errmsg string
+		var jobID int64
+		depsArg := sql.NullString{String: deps, Valid: deps != ""}
+		err := db.QueryRowContext(ctx, `
+			SELECT p_retval, p_errmsg, p_job_id FROM broker.broker_add_job(
+				$1, 'SELECT 1', 1, 0, 'sql', NULL, NULL, $2::BIGINT[], NULL, 3
+			)`, name, depsArg,
+		).Scan(&retval, &errmsg, &jobID)
+		require.NoError(t, err)
+		require.Equal(t, 0, retval, "broker_add_job(%s) failed: %s", name, errmsg)
+		return jobID
+	}
+
+	base := addJob("base", "")
+	dependent := addJob("dependent", "{"+sqlItoa(base)+"}")
+	require.NotZero(t, dependent)
+
+	// broker_get must not return "dependent" while "base" is still pending;
+	// it should return "base" instead.
+	var jobID sql.NullInt64
+	var getRetval int
+	var getErrmsg string
+	err := db.QueryRowContext(ctx,
+		"SELECT p_retval, p_errmsg, p_job_id FROM broker.broker_get($1, NULL, $2)", 1, 60,
+	).Scan(&getRetval, &getErrmsg, &jobID)
+	require.NoError(t, err)
+	require.Equal(t, 0, getRetval, getErrmsg)
+	require.True(t, jobID.Valid)
+	require.Equal(t, base, jobID.Int64, "dependency-free job must be claimed before its dependent")
+
+	// A self-dependency must be rejected by the table's CHECK constraint,
+	// regardless of caller.
+	target := addJob("self-target", "")
+	_, err = db.ExecContext(ctx,
+		"INSERT INTO broker.broker_job_dependency (job_id, depends_on_job_id) VALUES ($1, $1)", target,
+	)
+	require.Error(t, err, "self-dependency must violate the CHECK constraint")
+
+	// Re-adding the same dependency pair must be a no-op, not an error
+	// (ON CONFLICT DO NOTHING on the (job_id, depends_on_job_id) pair).
+	other := addJob("other-dep-target", "")
+	dependentTwo := addJob("dependent-two", "{"+sqlItoa(other)+"}")
+	_, err = db.ExecContext(ctx,
+		"INSERT INTO broker.broker_job_dependency (job_id, depends_on_job_id) VALUES ($1, $2) ON CONFLICT (job_id, depends_on_job_id) DO NOTHING",
+		dependentTwo, other,
+	)
+	require.NoError(t, err)
+
+	var depCount int
+	err = db.QueryRowContext(ctx,
+		"SELECT COUNT(*) FROM broker.broker_job_dependency WHERE job_id = $1 AND depends_on_job_id = $2",
+		dependentTwo, other,
+	).Scan(&depCount)
+	require.NoError(t, err)
+	require.Equal(t, 1, depCount, "duplicate dependency insert must not create a second row")
+}
+
+// TestStaleLeaseRecovery verifies that a job whose lease has expired while
+// still marked running is requeued (attempts remain) by
+// broker_recover_stale_jobs, and dead-lettered once attempts are exhausted.
+func TestStaleLeaseRecovery(t *testing.T) {
+	ctx := context.Background()
+	db := setupStage5Schema(t)
+
+	var jobID int64
+	var retval int
+	var errmsg string
+	err := db.QueryRowContext(ctx, `
+		SELECT p_retval, p_errmsg, p_job_id FROM broker.broker_add_job(
+			'stale-job', 'SELECT 1', 1, 0, 'sql', NULL, NULL, NULL, NULL, 2
+		)`,
+	).Scan(&retval, &errmsg, &jobID)
+	require.NoError(t, err)
+	require.Equal(t, 0, retval, errmsg)
+
+	// Simulate a worker having claimed the job with a lease that has
+	// already expired, without ever calling broker_run.
+	_, err = db.ExecContext(ctx, `
+		UPDATE broker.broker_jobs
+		SET complete_status = 1, attempt_count = 1,
+		    lease_token = gen_random_uuid(), leased_at = NOW() - INTERVAL '2 minutes',
+		    lease_expires_at = NOW() - INTERVAL '1 minute'
+		WHERE id_broker_jobs = $1`, jobID)
+	require.NoError(t, err)
+
+	var recRetval, recovered int
+	var recErrmsg string
+	err = db.QueryRowContext(ctx,
+		"SELECT p_retval, p_errmsg, p_recovered_count FROM broker.broker_recover_stale_jobs()",
+	).Scan(&recRetval, &recErrmsg, &recovered)
+	require.NoError(t, err)
+	require.Equal(t, 0, recRetval, recErrmsg)
+	require.Equal(t, 1, recovered)
+
+	var status int
+	var leaseToken sql.NullString
+	err = db.QueryRowContext(ctx,
+		"SELECT complete_status, lease_token FROM broker.broker_jobs WHERE id_broker_jobs = $1", jobID,
+	).Scan(&status, &leaseToken)
+	require.NoError(t, err)
+	require.Equal(t, 0, status, "job with attempts remaining must be requeued as pending")
+	require.False(t, leaseToken.Valid, "lease must be cleared on recovery")
+
+	// Exhaust attempts (attempt_count already 1, max_attempts 2) then
+	// simulate one more stale lease -- this time it must dead-letter.
+	_, err = db.ExecContext(ctx, `
+		UPDATE broker.broker_jobs
+		SET complete_status = 1, attempt_count = 2,
+		    lease_token = gen_random_uuid(), leased_at = NOW() - INTERVAL '2 minutes',
+		    lease_expires_at = NOW() - INTERVAL '1 minute'
+		WHERE id_broker_jobs = $1`, jobID)
+	require.NoError(t, err)
+
+	err = db.QueryRowContext(ctx,
+		"SELECT p_retval, p_errmsg, p_recovered_count FROM broker.broker_recover_stale_jobs()",
+	).Scan(&recRetval, &recErrmsg, &recovered)
+	require.NoError(t, err)
+	require.Equal(t, 0, recRetval, recErrmsg)
+	require.Equal(t, 1, recovered)
+
+	err = db.QueryRowContext(ctx,
+		"SELECT complete_status FROM broker.broker_jobs WHERE id_broker_jobs = $1", jobID,
+	).Scan(&status)
+	require.NoError(t, err)
+	require.Equal(t, 3, status, "job with attempts exhausted must be dead-lettered")
+}
+
+// TestFailedJobRetriesThenCompletesWithoutStranding verifies the original
+// bug fix: a job whose execution fails is requeued for retry (not stranded
+// in the running state) and, once it succeeds, ends up completed.
+func TestFailedJobRetriesThenCompletesWithoutStranding(t *testing.T) {
+	ctx := context.Background()
+	db := setupStage5Schema(t)
+
+	var jobID int64
+	var retval int
+	var errmsg string
+	err := db.QueryRowContext(ctx, `
+		SELECT p_retval, p_errmsg, p_job_id FROM broker.broker_add_job(
+			'flaky-job', 'SELECT 1/0', 1, 0, 'sql', NULL, NULL, NULL, NULL, 2
+		)`,
+	).Scan(&retval, &errmsg, &jobID)
+	require.NoError(t, err)
+	require.Equal(t, 0, retval, errmsg)
+
+	var claimedID sql.NullInt64
+	var leaseToken sql.NullString
+	var getRetval int
+	var getErrmsg string
+	err = db.QueryRowContext(ctx,
+		"SELECT p_retval, p_errmsg, p_job_id, p_lease_token FROM broker.broker_get($1, NULL, $2)", 1, 60,
+	).Scan(&getRetval, &getErrmsg, &claimedID, &leaseToken)
+	require.NoError(t, err)
+	require.Equal(t, 0, getRetval, getErrmsg)
+	require.True(t, claimedID.Valid)
+	require.Equal(t, jobID, claimedID.Int64)
+
+	var runRetval, jobStatus int
+	var runErrmsg string
+	err = db.QueryRowContext(ctx,
+		"SELECT p_retval, p_errmsg, p_job_status FROM broker.broker_run($1, $2)", jobID, leaseToken.String,
+	).Scan(&runRetval, &runErrmsg, &jobStatus)
+	require.NoError(t, err)
+	require.Equal(t, 0, runRetval, "broker_run must report success (retval=0) even when the job itself failed: %s", runErrmsg)
+	require.Equal(t, 0, jobStatus, "failing job with attempts remaining must be requeued (job_status=0), not stranded")
+
+	var status int
+	err = db.QueryRowContext(ctx,
+		"SELECT complete_status FROM broker.broker_jobs WHERE id_broker_jobs = $1", jobID,
+	).Scan(&status)
+	require.NoError(t, err)
+	require.Equal(t, 0, status, "job must be pending again, not stuck at running (1)")
+
+	// Presenting a stale lease token after the job was already reset must
+	// be rejected rather than silently re-running. broker_run returns early
+	// on the lease mismatch without ever setting p_job_status, so it comes
+	// back NULL here.
+	var staleJobStatus sql.NullInt64
+	err = db.QueryRowContext(ctx,
+		"SELECT p_retval, p_errmsg, p_job_status FROM broker.broker_run($1, $2)", jobID, leaseToken.String,
+	).Scan(&runRetval, &runErrmsg, &staleJobStatus)
+	require.NoError(t, err)
+	require.NotEqual(t, 0, runRetval, "broker_run must reject a stale/mismatched lease token")
+}
+
+func sqlItoa(v int64) string {
+	if v == 0 {
+		return "0"
+	}
+	neg := v < 0
+	if neg {
+		v = -v
+	}
+	var buf [20]byte
+	i := len(buf)
+	for v > 0 {
+		i--
+		buf[i] = byte('0' + v%10)
+		v /= 10
+	}
+	if neg {
+		i--
+		buf[i] = '-'
+	}
+	return string(buf[i:])
+}

tests/integration/testconfig_test.go

@@ -0,0 +1,26 @@
+package integration
+
+import (
+	"os"
+	"strconv"
+)
+
+// testDBHost and testDBPort let CI point the integration suite at a
+// dynamically-assigned Postgres (TEST_DB_HOST/TEST_DB_PORT), avoiding a fixed
+// host port that can collide with other jobs on a shared runner. Local dev
+// keeps working unset, defaulting to the docker-compose test stack.
+func testDBHost() string {
+	if h := os.Getenv("TEST_DB_HOST"); h != "" {
+		return h
+	}
+	return "127.0.0.1"
+}
+
+func testDBPort() int {
+	if p := os.Getenv("TEST_DB_PORT"); p != "" {
+		if n, err := strconv.Atoi(p); err == nil {
+			return n
+		}
+	}
+	return 5433
+}

tests/integration/workflow_test.go

@@ -3,6 +3,7 @@ package integration
 import (
 	"context"
 	"database/sql"
+	"fmt"
 	"log/slog"
 	"testing"
 	"time"
@@ -27,7 +28,7 @@ func TestBrokerWorkflow(t *testing.T) {
 	ctx := context.Background()
 
 	// Database connection string
-	connStr := "user=user password=password dbname=broker_test host=localhost port=5433 sslmode=disable"
+	connStr := fmt.Sprintf("user=user password=password dbname=broker_test host=%s port=%d sslmode=disable", testDBHost(), testDBPort())
 
 	// Connect to database with retry logic
 	db, err := connectWithRetry(connStr, 10, 2*time.Second)
@@ -43,8 +44,8 @@ func TestBrokerWorkflow(t *testing.T) {
 
 	// Create database adapter
 	postgresConfig := adapter.PostgresConfig{
-		Host:            "localhost",
+		Host:            testDBHost(),
-		Port:            5433,
+		Port:            testDBPort(),
 		Database:        "broker_test",
 		User:            "user",
 		Password:        "password",
@@ -65,8 +66,8 @@ func TestBrokerWorkflow(t *testing.T) {
 	// Install schema
 	t.Log("Installing database schema...")
 	installer := install.New(dbAdapter, logger)
-	err = installer.InstallSchema(ctx)
+	err = installer.ApplyMigrations(ctx)
-	require.NoError(t, err, "Failed to install schema")
+	require.NoError(t, err, "Failed to apply migrations")
 
 	// Verify installation
 	t.Log("Verifying schema installation...")
@@ -78,8 +79,8 @@ func TestBrokerWorkflow(t *testing.T) {
 		Databases: []config.DatabaseConfig{
 			{
 				Name:       "test_db",
-				Host:       "localhost",
+				Host:       testDBHost(),
-				Port:       5433,
+				Port:       testDBPort(),
 				Database:   "broker_test",
 				User:       "user",
 				Password:   "password",
@@ -93,7 +94,7 @@ func TestBrokerWorkflow(t *testing.T) {
 			QueueTimerSec:        1, // Short interval for testing
 			QueueBufferSize:      10,
 			WorkerIdleTimeoutSec: 5,
-			NotifyRetrySeconds:   5 * time.Second,
+			NotifyRetryInterval:  5 * time.Second,
 			EnableDebug:          true,
 		},
 		Logging: config.LoggingConfig{
@@ -127,7 +128,7 @@ func TestBrokerWorkflow(t *testing.T) {
 	var errmsg string
 	var jobID int64
 	err = db.QueryRowContext(ctx, `
-		SELECT * FROM broker_add_job(
+		SELECT * FROM broker.broker_add_job(
 			$1,  -- job_name
 			$2,  -- execute_str
 			$3,  -- job_queue
@@ -135,7 +136,9 @@ func TestBrokerWorkflow(t *testing.T) {
 			$5,  -- job_language
 			NULL, -- run_as
 			NULL, -- schedule_id
-			NULL  -- depends_on
+			NULL, -- depends_on_job_ids
+			NULL, -- idempotency_key
+			NULL  -- max_attempts
 		)
 	`,
 		"Test Job",
@@ -165,7 +168,7 @@ func TestBrokerWorkflow(t *testing.T) {
 			SELECT id_broker_jobs, job_name, job_priority, job_queue, job_language,
 			       execute_str, execute_result, error_msg, complete_status,
 			       created_at, updated_at
-			FROM broker_jobs
+			FROM broker.broker_jobs
 			WHERE id_broker_jobs = $1
 		`, jobID).Scan(
 			&job.ID,
@@ -243,32 +246,10 @@ func connectWithRetry(connStr string, maxRetries int, retryInterval time.Duratio
 	return nil, err
 }
 
-// cleanupSchema removes all broker tables and functions for a clean test
+// cleanupSchema drops the entire broker schema for a clean test run.
 func cleanupSchema(t *testing.T, db *sql.DB) {
-	tables := []string{"broker_jobs", "broker_queueinstance", "broker_schedule"}
+	_, err := db.Exec("DROP SCHEMA IF EXISTS broker CASCADE")
-	procedures := []string{
+	if err != nil {
-		"broker_get",
+		t.Logf("Warning: failed to drop broker schema: %v", err)
-		"broker_run",
-		"broker_set",
-		"broker_add_job",
-		"broker_register_instance",
-		"broker_ping_instance",
-		"broker_shutdown_instance",
-	}
-
-	// Drop procedures
-	for _, proc := range procedures {
-		_, err := db.Exec("DROP FUNCTION IF EXISTS " + proc + " CASCADE")
-		if err != nil {
-			t.Logf("Warning: failed to drop procedure %s: %v", proc, err)
-		}
-	}
-
-	// Drop tables
-	for _, table := range tables {
-		_, err := db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE")
-		if err != nil {
-			t.Logf("Warning: failed to drop table %s: %v", table, err)
-		}
 	}
 }