feat(broker): migrations-based install, roles, RLS, and job dependency groups

Replace the ad-hoc tables/procedures install layout with versioned, ordered SQL migrations tracked in broker_schema_migrations. Add optional least-privilege role provisioning (--with-roles), multi-tenant row-level security, lease-based job claiming with stale-lease recovery, and job dependencies -- both by job id and by fan-in job group. Add Docker/Compose support for running the broker and its test suite.
2026-09-17 22:09:41 +02:00
parent 602997bcdb
commit 4c8e1066d4
53 changed files with 2911 additions and 1016 deletions
@@ -0,0 +1,135 @@
+-- broker.broker_run
+-- Executes a job by its ID, presenting the lease token it was claimed with.
+--
+-- p_retval is reserved for infra failures (bad job id, job not found, wrong
+-- state, lease mismatch/expired, DB error). An executed-and-caught job
+-- failure is a *successful* invocation: p_retval stays 0 and the outcome is
+-- reported via p_job_status (0=requeued for retry, 2=completed, 3=dead-lettered)
+-- so the caller commits the terminal/retry state instead of rolling it back.
+--
+-- On failure, if attempt_count < max_attempts the job is reset to pending
+-- with exponential backoff (base 5s, capped at 300s); otherwise it is
+-- dead-lettered as complete_status = 3.
+
+CREATE OR REPLACE FUNCTION broker.broker_run(
+    p_job_id BIGINT,
+    p_lease_token UUID,
+    OUT p_retval INTEGER,
+    OUT p_errmsg TEXT,
+    OUT p_job_status INTEGER
+)
+RETURNS RECORD
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    v_job_record RECORD;
+    v_execute_result TEXT;
+    v_error_occurred BOOLEAN := false;
+    v_backoff_base CONSTANT INTEGER := 5;
+    v_backoff_cap CONSTANT INTEGER := 300;
+    v_backoff_secs INTEGER;
+BEGIN
+    p_retval := 0;
+    p_errmsg := '';
+    p_job_status := NULL;
+    v_execute_result := '';
+
+    IF p_job_id IS NULL OR p_job_id <= 0 THEN
+        p_retval := 1;
+        p_errmsg := 'Invalid job ID';
+        RETURN;
+    END IF;
+
+    SELECT id_broker_jobs, execute_str, job_language, complete_status, attempt_count, max_attempts, lease_token
+    INTO v_job_record
+    FROM broker.broker_jobs
+    WHERE id_broker_jobs = p_job_id
+    FOR UPDATE;
+
+    IF NOT FOUND THEN
+        p_retval := 2;
+        p_errmsg := 'Job not found';
+        RETURN;
+    END IF;
+
+    IF v_job_record.complete_status != 1 THEN
+        p_retval := 3;
+        p_errmsg := format('Job is not in running state (status: %s)', v_job_record.complete_status);
+        RETURN;
+    END IF;
+
+    IF v_job_record.lease_token IS DISTINCT FROM p_lease_token THEN
+        p_retval := 4;
+        p_errmsg := 'Lease token mismatch or expired; job was reclaimed by another worker';
+        RETURN;
+    END IF;
+
+    -- Execute the job
+    BEGIN
+        IF v_job_record.job_language IN ('sql', 'plpgsql') THEN
+            EXECUTE v_job_record.execute_str;
+            v_execute_result := 'Success';
+        ELSE
+            v_error_occurred := true;
+            v_execute_result := format('Unsupported job language: %s', v_job_record.job_language);
+        END IF;
+    EXCEPTION
+        WHEN OTHERS THEN
+            v_error_occurred := true;
+            v_execute_result := format('Error: %s', SQLERRM);
+    END;
+
+    IF v_error_occurred THEN
+        IF v_job_record.attempt_count < v_job_record.max_attempts THEN
+            v_backoff_secs := LEAST(POWER(2, v_job_record.attempt_count)::INTEGER * v_backoff_base, v_backoff_cap);
+
+            UPDATE broker.broker_jobs
+            SET complete_status = 0,  -- pending, retry
+                available_at = NOW() + make_interval(secs => v_backoff_secs),
+                error_msg = v_execute_result,
+                execute_result = v_execute_result,
+                lease_token = NULL,
+                leased_at = NULL,
+                lease_expires_at = NULL,
+                updated_at = NOW()
+            WHERE id_broker_jobs = p_job_id;
+
+            p_job_status := 0;
+        ELSE
+            UPDATE broker.broker_jobs
+            SET complete_status = 3,  -- failed (dead-letter, attempts exhausted)
+                error_msg = v_execute_result,
+                execute_result = v_execute_result,
+                lease_token = NULL,
+                leased_at = NULL,
+                lease_expires_at = NULL,
+                completed_at = NOW(),
+                updated_at = NOW()
+            WHERE id_broker_jobs = p_job_id;
+
+            p_job_status := 3;
+        END IF;
+    ELSE
+        UPDATE broker.broker_jobs
+        SET complete_status = 2,  -- completed
+            execute_result = v_execute_result,
+            error_msg = NULL,
+            lease_token = NULL,
+            leased_at = NULL,
+            lease_expires_at = NULL,
+            completed_at = NOW(),
+            updated_at = NOW()
+        WHERE id_broker_jobs = p_job_id;
+
+        p_job_status := 2;
+    END IF;
+
+EXCEPTION
+    WHEN OTHERS THEN
+        p_retval := 6;
+        p_errmsg := SQLERRM;
+        RAISE WARNING 'broker_run error: %', SQLERRM;
+END;
+$$;
+
+COMMENT ON FUNCTION broker.broker_run IS 'Executes a leased job; reports outcome via p_job_status without forcing a rollback of the terminal/retry state';