feat(broker): migrations-based install, roles, RLS, and job dependency groups
Replace the ad-hoc tables/procedures install layout with versioned, ordered SQL migrations tracked in broker_schema_migrations. Add optional least-privilege role provisioning (--with-roles), multi-tenant row-level security, lease-based job claiming with stale-lease recovery, and job dependencies -- both by job id and by fan-in job group. Add Docker/Compose support for running the broker and its test suite.
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
-- Dedicated schema for all broker objects.
|
||||
CREATE SCHEMA IF NOT EXISTS broker;
|
||||
REVOKE ALL ON SCHEMA broker FROM PUBLIC;
|
||||
|
||||
-- gen_random_uuid() for lease tokens.
|
||||
CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
||||
@@ -0,0 +1,25 @@
|
||||
-- broker.broker_queueinstance
|
||||
-- Tracks active and historical broker queue instances.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS broker.broker_queueinstance (
|
||||
id_broker_queueinstance BIGSERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
hostname VARCHAR(255) NOT NULL,
|
||||
pid INTEGER NOT NULL,
|
||||
version VARCHAR(50) NOT NULL,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'active',
|
||||
started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
last_ping_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
shutdown_at TIMESTAMP WITH TIME ZONE,
|
||||
queue_count INTEGER NOT NULL DEFAULT 0,
|
||||
jobs_handled BIGINT NOT NULL DEFAULT 0,
|
||||
|
||||
CONSTRAINT broker_queueinstance_status_check CHECK (status IN ('active', 'inactive', 'shutdown'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_queueinstance_status ON broker.broker_queueinstance(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_queueinstance_hostname ON broker.broker_queueinstance(hostname);
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_queueinstance_last_ping ON broker.broker_queueinstance(last_ping_at);
|
||||
|
||||
COMMENT ON TABLE broker.broker_queueinstance IS 'Tracks broker queue instances (active and historical). Single-active-instance-per-name is enforced via a pg_try_advisory_lock in broker_register_instance, not by this status column, which is observational only.';
|
||||
COMMENT ON COLUMN broker.broker_queueinstance.status IS 'Observational status: active, inactive, or shutdown. Ownership is enforced via advisory lock, not by reading this column.';
|
||||
@@ -0,0 +1,41 @@
|
||||
-- broker.broker_schedule
|
||||
-- Stores scheduled jobs (cron-like functionality).
|
||||
|
||||
CREATE TABLE IF NOT EXISTS broker.broker_schedule (
|
||||
id_broker_schedule BIGSERIAL PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL UNIQUE,
|
||||
cron_expr VARCHAR(100) NOT NULL,
|
||||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||||
job_name VARCHAR(255) NOT NULL,
|
||||
job_priority INTEGER NOT NULL DEFAULT 0,
|
||||
job_queue INTEGER NOT NULL DEFAULT 1,
|
||||
job_language VARCHAR(50) NOT NULL DEFAULT 'sql',
|
||||
execute_str TEXT NOT NULL,
|
||||
run_as VARCHAR(100),
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
last_run_at TIMESTAMP WITH TIME ZONE,
|
||||
next_run_at TIMESTAMP WITH TIME ZONE,
|
||||
|
||||
CONSTRAINT broker_schedule_job_queue_check CHECK (job_queue > 0)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_schedule_enabled ON broker.broker_schedule(enabled);
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_schedule_next_run ON broker.broker_schedule(next_run_at) WHERE enabled = true;
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_schedule_name ON broker.broker_schedule(name);
|
||||
|
||||
COMMENT ON TABLE broker.broker_schedule IS 'Scheduled jobs (cron-like functionality)';
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.tf_broker_schedule_update_timestamp()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS t_broker_schedule_updated_at ON broker.broker_schedule;
|
||||
CREATE TRIGGER t_broker_schedule_updated_at
|
||||
BEFORE UPDATE ON broker.broker_schedule
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION broker.tf_broker_schedule_update_timestamp();
|
||||
@@ -0,0 +1,93 @@
|
||||
-- broker.broker_jobs
|
||||
-- Job queue for broker execution.
|
||||
-- tenant_id / RLS: rows are only visible/writable when tenant_id matches
|
||||
-- current_setting('broker.tenant_id', true) for the current transaction.
|
||||
-- Callers must invoke broker.broker_set_tenant(...) before enqueue/claim;
|
||||
-- if they don't, tenant_id defaults to 'default' and current_setting
|
||||
-- also defaults to NULL -> broker_add_job coalesces to 'default' so
|
||||
-- single-tenant use keeps working unmodified.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS broker.broker_jobs (
|
||||
id_broker_jobs BIGSERIAL PRIMARY KEY,
|
||||
job_name VARCHAR(255) NOT NULL,
|
||||
job_priority INTEGER NOT NULL DEFAULT 0,
|
||||
job_queue INTEGER NOT NULL DEFAULT 1,
|
||||
job_language VARCHAR(50) NOT NULL DEFAULT 'sql',
|
||||
execute_str TEXT NOT NULL,
|
||||
execute_result TEXT,
|
||||
error_msg TEXT,
|
||||
complete_status INTEGER NOT NULL DEFAULT 0,
|
||||
run_as VARCHAR(100),
|
||||
rid_broker_schedule BIGINT,
|
||||
rid_broker_queueinstance BIGINT,
|
||||
tenant_id TEXT NOT NULL DEFAULT 'default',
|
||||
|
||||
-- Lease / retry / idempotency
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||||
max_attempts INTEGER NOT NULL DEFAULT 1,
|
||||
available_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
leased_at TIMESTAMP WITH TIME ZONE,
|
||||
lease_expires_at TIMESTAMP WITH TIME ZONE,
|
||||
lease_token UUID,
|
||||
idempotency_key TEXT,
|
||||
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
started_at TIMESTAMP WITH TIME ZONE,
|
||||
completed_at TIMESTAMP WITH TIME ZONE,
|
||||
|
||||
CONSTRAINT broker_jobs_complete_status_check CHECK (complete_status IN (0, 1, 2, 3, 4)),
|
||||
CONSTRAINT broker_jobs_job_queue_check CHECK (job_queue > 0),
|
||||
CONSTRAINT fk_schedule FOREIGN KEY (rid_broker_schedule) REFERENCES broker.broker_schedule(id_broker_schedule) ON DELETE SET NULL,
|
||||
CONSTRAINT fk_instance FOREIGN KEY (rid_broker_queueinstance) REFERENCES broker.broker_queueinstance(id_broker_queueinstance) ON DELETE SET NULL
|
||||
);
|
||||
|
||||
-- General-purpose indexes
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_jobs_status ON broker.broker_jobs(complete_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_jobs_schedule ON broker.broker_jobs(rid_broker_schedule);
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_jobs_instance ON broker.broker_jobs(rid_broker_queueinstance);
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_jobs_created ON broker.broker_jobs(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_jobs_name ON broker.broker_jobs(job_name, complete_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_jobs_tenant ON broker.broker_jobs(tenant_id);
|
||||
|
||||
-- Claim index: exactly what broker_get's WHERE/ORDER BY needs, partial on pending rows only.
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_jobs_claim
|
||||
ON broker.broker_jobs (job_queue, job_priority DESC, created_at, id_broker_jobs)
|
||||
WHERE complete_status = 0;
|
||||
|
||||
-- Idempotency: at most one pending/any job per (queue, key) when a key is supplied.
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_broker_jobs_idempotency
|
||||
ON broker.broker_jobs (job_queue, idempotency_key)
|
||||
WHERE idempotency_key IS NOT NULL;
|
||||
|
||||
COMMENT ON TABLE broker.broker_jobs IS 'Job queue for broker execution';
|
||||
COMMENT ON COLUMN broker.broker_jobs.complete_status IS '0=pending, 1=running, 2=completed, 3=failed (terminal or dead-lettered once attempt_count>=max_attempts), 4=cancelled';
|
||||
COMMENT ON COLUMN broker.broker_jobs.tenant_id IS 'RLS tenant scaffold; defaults to ''default'' for single-tenant use';
|
||||
COMMENT ON COLUMN broker.broker_jobs.attempt_count IS 'Number of times this job has been claimed/executed';
|
||||
COMMENT ON COLUMN broker.broker_jobs.max_attempts IS 'Job is dead-lettered (failed) once attempt_count reaches this value';
|
||||
COMMENT ON COLUMN broker.broker_jobs.available_at IS 'Job is not claimable until now() >= available_at (used for retry backoff)';
|
||||
COMMENT ON COLUMN broker.broker_jobs.lease_token IS 'Token handed out by broker_get; broker_run requires a matching token to execute, so an expired/reclaimed lease cannot be double-processed';
|
||||
COMMENT ON COLUMN broker.broker_jobs.idempotency_key IS 'Optional caller-supplied key; unique per (job_queue, idempotency_key)';
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.tf_broker_jobs_update_timestamp()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = NOW();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS t_broker_jobs_updated_at ON broker.broker_jobs;
|
||||
CREATE TRIGGER t_broker_jobs_updated_at
|
||||
BEFORE UPDATE ON broker.broker_jobs
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION broker.tf_broker_jobs_update_timestamp();
|
||||
|
||||
-- Row Level Security: tenant isolation
|
||||
ALTER TABLE broker.broker_jobs ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE broker.broker_jobs FORCE ROW LEVEL SECURITY;
|
||||
|
||||
DROP POLICY IF EXISTS broker_jobs_tenant_isolation ON broker.broker_jobs;
|
||||
CREATE POLICY broker_jobs_tenant_isolation ON broker.broker_jobs
|
||||
USING (tenant_id = COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default'))
|
||||
WITH CHECK (tenant_id = COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default'));
|
||||
@@ -0,0 +1,27 @@
|
||||
-- broker.broker_job_dependency
|
||||
-- Replaces the old broker_jobs.depends_on text[] column: job_id is only
|
||||
-- claimable once every row it depends on has complete_status = 2 (completed).
|
||||
|
||||
CREATE TABLE IF NOT EXISTS broker.broker_job_dependency (
|
||||
job_id BIGINT NOT NULL REFERENCES broker.broker_jobs(id_broker_jobs) ON DELETE CASCADE,
|
||||
depends_on_job_id BIGINT NOT NULL REFERENCES broker.broker_jobs(id_broker_jobs) ON DELETE CASCADE,
|
||||
tenant_id TEXT NOT NULL DEFAULT 'default',
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (job_id, depends_on_job_id),
|
||||
CHECK (job_id <> depends_on_job_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_job_dependency_reverse
|
||||
ON broker.broker_job_dependency (depends_on_job_id, job_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_job_dependency_tenant
|
||||
ON broker.broker_job_dependency (tenant_id);
|
||||
|
||||
COMMENT ON TABLE broker.broker_job_dependency IS 'job_id is not claimable until every depends_on_job_id row has complete_status = 2 (completed)';
|
||||
|
||||
ALTER TABLE broker.broker_job_dependency ENABLE ROW LEVEL SECURITY;
|
||||
ALTER TABLE broker.broker_job_dependency FORCE ROW LEVEL SECURITY;
|
||||
|
||||
DROP POLICY IF EXISTS broker_job_dependency_tenant_isolation ON broker.broker_job_dependency;
|
||||
CREATE POLICY broker_job_dependency_tenant_isolation ON broker.broker_job_dependency
|
||||
USING (tenant_id = COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default'))
|
||||
WITH CHECK (tenant_id = COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default'));
|
||||
@@ -0,0 +1,13 @@
|
||||
-- broker.broker_set_tenant
|
||||
-- Sets the RLS tenant context for the current transaction (SET LOCAL semantics
|
||||
-- via set_config(..., true)). Callers must invoke this before enqueue/claim
|
||||
-- if they are not using the 'default' tenant.
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_set_tenant(p_tenant_id TEXT)
|
||||
RETURNS VOID
|
||||
LANGUAGE SQL
|
||||
AS $$
|
||||
SELECT set_config('broker.tenant_id', p_tenant_id, true);
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION broker.broker_set_tenant IS 'Sets broker.tenant_id for the current transaction only (SET LOCAL semantics)';
|
||||
@@ -0,0 +1,83 @@
|
||||
-- broker.broker_get
|
||||
-- Claims the next eligible job from a queue: pending, available (backoff
|
||||
-- elapsed), no incomplete dependency, and visible under the caller's RLS
|
||||
-- tenant. Grants a lease (lease_token) that broker_run must present back.
|
||||
-- Returns: p_retval (0=success, >0=infra error), p_errmsg, p_job_id, p_lease_token.
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_get(
|
||||
p_queue_number INTEGER,
|
||||
p_instance_id BIGINT DEFAULT NULL,
|
||||
p_lease_seconds INTEGER DEFAULT 60,
|
||||
OUT p_retval INTEGER,
|
||||
OUT p_errmsg TEXT,
|
||||
OUT p_job_id BIGINT,
|
||||
OUT p_lease_token UUID
|
||||
)
|
||||
RETURNS RECORD
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
v_job_id BIGINT;
|
||||
v_lease_token UUID;
|
||||
BEGIN
|
||||
p_retval := 0;
|
||||
p_errmsg := '';
|
||||
p_job_id := NULL;
|
||||
p_lease_token := NULL;
|
||||
|
||||
IF p_queue_number IS NULL OR p_queue_number <= 0 THEN
|
||||
p_retval := 1;
|
||||
p_errmsg := 'Invalid queue number';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF p_lease_seconds IS NULL OR p_lease_seconds <= 0 THEN
|
||||
p_lease_seconds := 60;
|
||||
END IF;
|
||||
|
||||
SELECT candidate.id_broker_jobs
|
||||
INTO v_job_id
|
||||
FROM broker.broker_jobs candidate
|
||||
WHERE candidate.job_queue = p_queue_number
|
||||
AND candidate.complete_status = 0
|
||||
AND candidate.available_at <= NOW()
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM broker.broker_job_dependency d
|
||||
JOIN broker.broker_jobs dep ON dep.id_broker_jobs = d.depends_on_job_id
|
||||
WHERE d.job_id = candidate.id_broker_jobs
|
||||
AND dep.complete_status <> 2
|
||||
)
|
||||
ORDER BY candidate.job_priority DESC, candidate.created_at ASC, candidate.id_broker_jobs ASC
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
v_lease_token := gen_random_uuid();
|
||||
|
||||
UPDATE broker.broker_jobs
|
||||
SET complete_status = 1, -- running
|
||||
started_at = NOW(),
|
||||
rid_broker_queueinstance = p_instance_id,
|
||||
attempt_count = attempt_count + 1,
|
||||
lease_token = v_lease_token,
|
||||
leased_at = NOW(),
|
||||
lease_expires_at = NOW() + make_interval(secs => p_lease_seconds),
|
||||
updated_at = NOW()
|
||||
WHERE id_broker_jobs = v_job_id;
|
||||
|
||||
p_job_id := v_job_id;
|
||||
p_lease_token := v_lease_token;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 2;
|
||||
p_errmsg := SQLERRM;
|
||||
RAISE WARNING 'broker_get error: %', SQLERRM;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION broker.broker_get IS 'Claims the next eligible job from a queue and grants a lease';
|
||||
@@ -0,0 +1,135 @@
|
||||
-- broker.broker_run
|
||||
-- Executes a job by its ID, presenting the lease token it was claimed with.
|
||||
--
|
||||
-- p_retval is reserved for infra failures (bad job id, job not found, wrong
|
||||
-- state, lease mismatch/expired, DB error). An executed-and-caught job
|
||||
-- failure is a *successful* invocation: p_retval stays 0 and the outcome is
|
||||
-- reported via p_job_status (0=requeued for retry, 2=completed, 3=dead-lettered)
|
||||
-- so the caller commits the terminal/retry state instead of rolling it back.
|
||||
--
|
||||
-- On failure, if attempt_count < max_attempts the job is reset to pending
|
||||
-- with exponential backoff (base 5s, capped at 300s); otherwise it is
|
||||
-- dead-lettered as complete_status = 3.
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_run(
|
||||
p_job_id BIGINT,
|
||||
p_lease_token UUID,
|
||||
OUT p_retval INTEGER,
|
||||
OUT p_errmsg TEXT,
|
||||
OUT p_job_status INTEGER
|
||||
)
|
||||
RETURNS RECORD
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
v_job_record RECORD;
|
||||
v_execute_result TEXT;
|
||||
v_error_occurred BOOLEAN := false;
|
||||
v_backoff_base CONSTANT INTEGER := 5;
|
||||
v_backoff_cap CONSTANT INTEGER := 300;
|
||||
v_backoff_secs INTEGER;
|
||||
BEGIN
|
||||
p_retval := 0;
|
||||
p_errmsg := '';
|
||||
p_job_status := NULL;
|
||||
v_execute_result := '';
|
||||
|
||||
IF p_job_id IS NULL OR p_job_id <= 0 THEN
|
||||
p_retval := 1;
|
||||
p_errmsg := 'Invalid job ID';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT id_broker_jobs, execute_str, job_language, complete_status, attempt_count, max_attempts, lease_token
|
||||
INTO v_job_record
|
||||
FROM broker.broker_jobs
|
||||
WHERE id_broker_jobs = p_job_id
|
||||
FOR UPDATE;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
p_retval := 2;
|
||||
p_errmsg := 'Job not found';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF v_job_record.complete_status != 1 THEN
|
||||
p_retval := 3;
|
||||
p_errmsg := format('Job is not in running state (status: %s)', v_job_record.complete_status);
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF v_job_record.lease_token IS DISTINCT FROM p_lease_token THEN
|
||||
p_retval := 4;
|
||||
p_errmsg := 'Lease token mismatch or expired; job was reclaimed by another worker';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
-- Execute the job
|
||||
BEGIN
|
||||
IF v_job_record.job_language IN ('sql', 'plpgsql') THEN
|
||||
EXECUTE v_job_record.execute_str;
|
||||
v_execute_result := 'Success';
|
||||
ELSE
|
||||
v_error_occurred := true;
|
||||
v_execute_result := format('Unsupported job language: %s', v_job_record.job_language);
|
||||
END IF;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
v_error_occurred := true;
|
||||
v_execute_result := format('Error: %s', SQLERRM);
|
||||
END;
|
||||
|
||||
IF v_error_occurred THEN
|
||||
IF v_job_record.attempt_count < v_job_record.max_attempts THEN
|
||||
v_backoff_secs := LEAST(POWER(2, v_job_record.attempt_count)::INTEGER * v_backoff_base, v_backoff_cap);
|
||||
|
||||
UPDATE broker.broker_jobs
|
||||
SET complete_status = 0, -- pending, retry
|
||||
available_at = NOW() + make_interval(secs => v_backoff_secs),
|
||||
error_msg = v_execute_result,
|
||||
execute_result = v_execute_result,
|
||||
lease_token = NULL,
|
||||
leased_at = NULL,
|
||||
lease_expires_at = NULL,
|
||||
updated_at = NOW()
|
||||
WHERE id_broker_jobs = p_job_id;
|
||||
|
||||
p_job_status := 0;
|
||||
ELSE
|
||||
UPDATE broker.broker_jobs
|
||||
SET complete_status = 3, -- failed (dead-letter, attempts exhausted)
|
||||
error_msg = v_execute_result,
|
||||
execute_result = v_execute_result,
|
||||
lease_token = NULL,
|
||||
leased_at = NULL,
|
||||
lease_expires_at = NULL,
|
||||
completed_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE id_broker_jobs = p_job_id;
|
||||
|
||||
p_job_status := 3;
|
||||
END IF;
|
||||
ELSE
|
||||
UPDATE broker.broker_jobs
|
||||
SET complete_status = 2, -- completed
|
||||
execute_result = v_execute_result,
|
||||
error_msg = NULL,
|
||||
lease_token = NULL,
|
||||
leased_at = NULL,
|
||||
lease_expires_at = NULL,
|
||||
completed_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE id_broker_jobs = p_job_id;
|
||||
|
||||
p_job_status := 2;
|
||||
END IF;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 6;
|
||||
p_errmsg := SQLERRM;
|
||||
RAISE WARNING 'broker_run error: %', SQLERRM;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION broker.broker_run IS 'Executes a leased job; reports outcome via p_job_status without forcing a rollback of the terminal/retry state';
|
||||
@@ -0,0 +1,65 @@
|
||||
-- broker.broker_set
|
||||
-- Minimal whitelist of session options. The previous SET SESSION AUTHORIZATION
|
||||
-- and search_path branches were removed: they let a caller assume an arbitrary
|
||||
-- Postgres role or schema search order from inside a plpgsql function with no
|
||||
-- identity model behind it, which is unsafe and was unused.
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_set(
|
||||
p_option_name TEXT,
|
||||
p_option_value TEXT,
|
||||
OUT p_retval INTEGER,
|
||||
OUT p_errmsg TEXT
|
||||
)
|
||||
RETURNS RECORD
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
v_sql TEXT;
|
||||
BEGIN
|
||||
p_retval := 0;
|
||||
p_errmsg := '';
|
||||
|
||||
IF p_option_name IS NULL OR p_option_name = '' THEN
|
||||
p_retval := 1;
|
||||
p_errmsg := 'Option name is required';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
CASE LOWER(p_option_name)
|
||||
WHEN 'application_name' THEN
|
||||
BEGIN
|
||||
v_sql := format('SET LOCAL application_name TO %L', p_option_value);
|
||||
EXECUTE v_sql;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 3;
|
||||
p_errmsg := format('Failed to set application_name: %s', SQLERRM);
|
||||
RETURN;
|
||||
END;
|
||||
|
||||
WHEN 'timezone' THEN
|
||||
BEGIN
|
||||
v_sql := format('SET LOCAL timezone TO %L', p_option_value);
|
||||
EXECUTE v_sql;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 5;
|
||||
p_errmsg := format('Failed to set timezone: %s', SQLERRM);
|
||||
RETURN;
|
||||
END;
|
||||
|
||||
ELSE
|
||||
p_retval := 10;
|
||||
p_errmsg := format('Unknown option: %s', p_option_name);
|
||||
RETURN;
|
||||
END CASE;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 99;
|
||||
p_errmsg := SQLERRM;
|
||||
RAISE WARNING 'broker_set error: %', SQLERRM;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION broker.broker_set IS 'Sets a whitelisted session-local option (application_name, timezone)';
|
||||
@@ -0,0 +1,64 @@
|
||||
-- broker.broker_register_instance
|
||||
-- Registers a broker instance, using a session-scoped advisory lock keyed by
|
||||
-- name to guarantee only one active instance per name -- no race window, no
|
||||
-- "check COUNT(*) then insert" gap. The caller MUST run this on a pinned
|
||||
-- connection it keeps open for the process lifetime (Go: db.Conn(ctx)) and
|
||||
-- release the lock (pg_advisory_unlock) itself on shutdown, since the lock
|
||||
-- lives with the backend session, not with the row.
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_register_instance(
|
||||
p_name TEXT,
|
||||
p_hostname TEXT,
|
||||
p_pid INTEGER,
|
||||
p_version TEXT,
|
||||
p_queue_count INTEGER,
|
||||
OUT p_retval INTEGER,
|
||||
OUT p_errmsg TEXT,
|
||||
OUT p_instance_id BIGINT
|
||||
)
|
||||
RETURNS RECORD
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
v_lock_key BIGINT;
|
||||
BEGIN
|
||||
p_retval := 0;
|
||||
p_errmsg := '';
|
||||
p_instance_id := NULL;
|
||||
|
||||
IF p_name IS NULL OR p_name = '' THEN
|
||||
p_retval := 1;
|
||||
p_errmsg := 'Instance name is required';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF p_hostname IS NULL OR p_hostname = '' THEN
|
||||
p_retval := 2;
|
||||
p_errmsg := 'Hostname is required';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
v_lock_key := hashtextextended('broker:' || p_name, 0);
|
||||
|
||||
IF NOT pg_try_advisory_lock(v_lock_key) THEN
|
||||
p_retval := 3;
|
||||
p_errmsg := 'Another broker instance is already active for this name (advisory lock held). Only one broker instance per name is allowed.';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
INSERT INTO broker.broker_queueinstance (
|
||||
name, hostname, pid, version, status, queue_count, started_at, last_ping_at
|
||||
) VALUES (
|
||||
p_name, p_hostname, p_pid, p_version, 'active', p_queue_count, NOW(), NOW()
|
||||
)
|
||||
RETURNING id_broker_queueinstance INTO p_instance_id;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 99;
|
||||
p_errmsg := SQLERRM;
|
||||
RAISE WARNING 'broker_register_instance error: %', SQLERRM;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION broker.broker_register_instance IS 'Registers a broker instance; caller must hold the connection open for the process lifetime (advisory lock is session-scoped)';
|
||||
@@ -0,0 +1,137 @@
|
||||
-- broker.broker_add_job
|
||||
-- Adds a new job (optionally with dependencies and an idempotency key) and
|
||||
-- sends a wake-only NOTIFY -- the payload carries only the queue number
|
||||
-- (job id kept solely for logging); workers re-claim via broker_get rather
|
||||
-- than executing the notified row directly, so a notification can never
|
||||
-- hand a job to a worker before it's actually claimable.
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_add_job(
|
||||
p_job_name TEXT,
|
||||
p_execute_str TEXT,
|
||||
p_job_queue INTEGER DEFAULT 1,
|
||||
p_job_priority INTEGER DEFAULT 0,
|
||||
p_job_language TEXT DEFAULT 'sql',
|
||||
p_run_as TEXT DEFAULT NULL,
|
||||
p_schedule_id BIGINT DEFAULT NULL,
|
||||
p_depends_on_job_ids BIGINT[] DEFAULT NULL,
|
||||
p_idempotency_key TEXT DEFAULT NULL,
|
||||
p_max_attempts INTEGER DEFAULT 1,
|
||||
OUT p_retval INTEGER,
|
||||
OUT p_errmsg TEXT,
|
||||
OUT p_job_id BIGINT
|
||||
)
|
||||
RETURNS RECORD
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
v_notification_payload JSON;
|
||||
v_tenant_id TEXT;
|
||||
v_dep_id BIGINT;
|
||||
v_cycle_exists BOOLEAN;
|
||||
BEGIN
|
||||
p_retval := 0;
|
||||
p_errmsg := '';
|
||||
p_job_id := NULL;
|
||||
|
||||
IF p_job_name IS NULL OR p_job_name = '' THEN
|
||||
p_retval := 1;
|
||||
p_errmsg := 'Job name is required';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF p_execute_str IS NULL OR p_execute_str = '' THEN
|
||||
p_retval := 2;
|
||||
p_errmsg := 'Execute string is required';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF p_job_queue IS NULL OR p_job_queue <= 0 THEN
|
||||
p_retval := 3;
|
||||
p_errmsg := 'Invalid job queue number';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF p_max_attempts IS NULL OR p_max_attempts <= 0 THEN
|
||||
p_max_attempts := 1;
|
||||
END IF;
|
||||
|
||||
-- Falls back to 'default' when the caller never called broker_set_tenant,
|
||||
-- so single-tenant use (and the RLS WITH CHECK on insert) keeps working.
|
||||
v_tenant_id := COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default');
|
||||
|
||||
IF p_idempotency_key IS NOT NULL THEN
|
||||
SELECT id_broker_jobs INTO p_job_id
|
||||
FROM broker.broker_jobs
|
||||
WHERE job_queue = p_job_queue
|
||||
AND idempotency_key = p_idempotency_key
|
||||
AND tenant_id = v_tenant_id;
|
||||
|
||||
IF FOUND THEN
|
||||
p_errmsg := 'Job with this idempotency key already exists; returning existing job id';
|
||||
RETURN;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
INSERT INTO broker.broker_jobs (
|
||||
job_name, job_priority, job_queue, job_language, execute_str, run_as,
|
||||
rid_broker_schedule, tenant_id, max_attempts, idempotency_key, complete_status
|
||||
) VALUES (
|
||||
p_job_name, p_job_priority, p_job_queue, p_job_language, p_execute_str, p_run_as,
|
||||
p_schedule_id, v_tenant_id, p_max_attempts, p_idempotency_key, 0
|
||||
)
|
||||
RETURNING id_broker_jobs INTO p_job_id;
|
||||
|
||||
IF p_depends_on_job_ids IS NOT NULL THEN
|
||||
FOREACH v_dep_id IN ARRAY p_depends_on_job_ids LOOP
|
||||
IF v_dep_id IS NULL THEN
|
||||
CONTINUE;
|
||||
END IF;
|
||||
|
||||
IF v_dep_id = p_job_id THEN
|
||||
p_retval := 20;
|
||||
p_errmsg := 'Invalid dependency: a job cannot depend on itself';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM broker.broker_job_dependency
|
||||
WHERE job_id = v_dep_id AND depends_on_job_id = p_job_id
|
||||
) INTO v_cycle_exists;
|
||||
|
||||
IF v_cycle_exists THEN
|
||||
p_retval := 21;
|
||||
p_errmsg := format('Invalid dependency: job %s already depends on %s (would create a cycle)', v_dep_id, p_job_id);
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
INSERT INTO broker.broker_job_dependency (job_id, depends_on_job_id, tenant_id)
|
||||
VALUES (p_job_id, v_dep_id, v_tenant_id)
|
||||
ON CONFLICT (job_id, depends_on_job_id) DO NOTHING;
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
v_notification_payload := json_build_object(
|
||||
'queue', p_job_queue,
|
||||
'job_id', p_job_id
|
||||
);
|
||||
|
||||
PERFORM pg_notify('broker.event', v_notification_payload::text);
|
||||
|
||||
EXCEPTION
|
||||
WHEN unique_violation THEN
|
||||
-- Concurrent insert raced us to the same idempotency key.
|
||||
p_retval := 0;
|
||||
p_errmsg := 'Job with this idempotency key already exists; returning existing job id';
|
||||
SELECT id_broker_jobs INTO p_job_id
|
||||
FROM broker.broker_jobs
|
||||
WHERE job_queue = p_job_queue
|
||||
AND idempotency_key = p_idempotency_key
|
||||
AND tenant_id = v_tenant_id;
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 99;
|
||||
p_errmsg := SQLERRM;
|
||||
RAISE WARNING 'broker_add_job error: %', SQLERRM;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION broker.broker_add_job IS 'Adds a job (with optional dependencies/idempotency key) and sends a wake-only NOTIFY';
|
||||
@@ -0,0 +1,83 @@
|
||||
-- broker.broker_ping_instance / broker.broker_shutdown_instance
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_ping_instance(
|
||||
p_instance_id BIGINT,
|
||||
p_jobs_handled BIGINT DEFAULT NULL,
|
||||
OUT p_retval INTEGER,
|
||||
OUT p_errmsg TEXT
|
||||
)
|
||||
RETURNS RECORD
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
p_retval := 0;
|
||||
p_errmsg := '';
|
||||
|
||||
IF p_instance_id IS NULL OR p_instance_id <= 0 THEN
|
||||
p_retval := 1;
|
||||
p_errmsg := 'Invalid instance ID';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF p_jobs_handled IS NOT NULL THEN
|
||||
UPDATE broker.broker_queueinstance
|
||||
SET last_ping_at = NOW(), jobs_handled = p_jobs_handled
|
||||
WHERE id_broker_queueinstance = p_instance_id;
|
||||
ELSE
|
||||
UPDATE broker.broker_queueinstance
|
||||
SET last_ping_at = NOW()
|
||||
WHERE id_broker_queueinstance = p_instance_id;
|
||||
END IF;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
p_retval := 2;
|
||||
p_errmsg := 'Instance not found';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 99;
|
||||
p_errmsg := SQLERRM;
|
||||
RAISE WARNING 'broker_ping_instance error: %', SQLERRM;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_shutdown_instance(
|
||||
p_instance_id BIGINT,
|
||||
OUT p_retval INTEGER,
|
||||
OUT p_errmsg TEXT
|
||||
)
|
||||
RETURNS RECORD
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
p_retval := 0;
|
||||
p_errmsg := '';
|
||||
|
||||
IF p_instance_id IS NULL OR p_instance_id <= 0 THEN
|
||||
p_retval := 1;
|
||||
p_errmsg := 'Invalid instance ID';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
UPDATE broker.broker_queueinstance
|
||||
SET status = 'shutdown', shutdown_at = NOW()
|
||||
WHERE id_broker_queueinstance = p_instance_id;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
p_retval := 2;
|
||||
p_errmsg := 'Instance not found';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 99;
|
||||
p_errmsg := SQLERRM;
|
||||
RAISE WARNING 'broker_shutdown_instance error: %', SQLERRM;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION broker.broker_ping_instance IS 'Updates the last ping timestamp for an instance';
|
||||
COMMENT ON FUNCTION broker.broker_shutdown_instance IS 'Marks an instance as shutdown (does not release the advisory lock -- caller must pg_advisory_unlock on its pinned connection)';
|
||||
@@ -0,0 +1,66 @@
|
||||
-- broker.broker_recover_stale_jobs
|
||||
-- Recovers jobs whose lease has expired while still 'running' (a worker died
|
||||
-- or was killed mid-execution without updating status). Applies the same
|
||||
-- retry/backoff rule as broker_run: retry while attempts remain, otherwise
|
||||
-- dead-letter. Marked SECURITY DEFINER so the sweep runs across all tenants
|
||||
-- regardless of caller: this requires the function's owner (whichever role
|
||||
-- runs the migrations, intended to be broker_admin) to have BYPASSRLS --
|
||||
-- broker_jobs/broker_job_dependency use FORCE ROW LEVEL SECURITY, so without
|
||||
-- BYPASSRLS on the owner this would silently only ever see the empty/no
|
||||
-- tenant context.
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_recover_stale_jobs(
|
||||
OUT p_retval INTEGER,
|
||||
OUT p_errmsg TEXT,
|
||||
OUT p_recovered_count INTEGER
|
||||
)
|
||||
RETURNS RECORD
|
||||
LANGUAGE plpgsql
|
||||
SECURITY DEFINER
|
||||
SET search_path = broker, pg_temp
|
||||
AS $$
|
||||
DECLARE
|
||||
v_backoff_base CONSTANT INTEGER := 5;
|
||||
v_backoff_cap CONSTANT INTEGER := 300;
|
||||
BEGIN
|
||||
p_retval := 0;
|
||||
p_errmsg := '';
|
||||
p_recovered_count := 0;
|
||||
|
||||
WITH stale AS (
|
||||
SELECT id_broker_jobs, attempt_count, max_attempts
|
||||
FROM broker.broker_jobs
|
||||
WHERE complete_status = 1
|
||||
AND lease_expires_at IS NOT NULL
|
||||
AND lease_expires_at < NOW()
|
||||
FOR UPDATE SKIP LOCKED
|
||||
),
|
||||
recovered AS (
|
||||
UPDATE broker.broker_jobs j
|
||||
SET complete_status = CASE WHEN s.attempt_count < s.max_attempts THEN 0 ELSE 3 END,
|
||||
available_at = CASE
|
||||
WHEN s.attempt_count < s.max_attempts
|
||||
THEN NOW() + make_interval(secs => LEAST(POWER(2, s.attempt_count)::INTEGER * v_backoff_base, v_backoff_cap))
|
||||
ELSE j.available_at
|
||||
END,
|
||||
error_msg = CASE WHEN s.attempt_count >= s.max_attempts THEN COALESCE(j.error_msg, 'Lease expired and max attempts exhausted') ELSE j.error_msg END,
|
||||
completed_at = CASE WHEN s.attempt_count >= s.max_attempts THEN NOW() ELSE j.completed_at END,
|
||||
lease_token = NULL,
|
||||
leased_at = NULL,
|
||||
lease_expires_at = NULL,
|
||||
updated_at = NOW()
|
||||
FROM stale s
|
||||
WHERE j.id_broker_jobs = s.id_broker_jobs
|
||||
RETURNING j.id_broker_jobs
|
||||
)
|
||||
SELECT COUNT(*) INTO p_recovered_count FROM recovered;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 99;
|
||||
p_errmsg := SQLERRM;
|
||||
RAISE WARNING 'broker_recover_stale_jobs error: %', SQLERRM;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION broker.broker_recover_stale_jobs IS 'Requeues (or dead-letters) jobs whose lease expired while still running';
|
||||
@@ -0,0 +1,41 @@
|
||||
-- Adds job groups: every job belongs to a job_group (defaults to its own
|
||||
-- job_name when not given explicitly, set by broker_add_job). A dependency
|
||||
-- can now target a whole group instead of a single job id -- the dependent
|
||||
-- job is claimable once every job tagged with that group has completed
|
||||
-- (complete_status = 2); already-completed group members simply drop out of
|
||||
-- the gating check, they don't need to have existed at any particular time.
|
||||
-- The existing id-based dependency (broker_job_dependency.depends_on_job_id)
|
||||
-- is kept as-is; each dependency row targets exactly one of an id or a group.
|
||||
|
||||
ALTER TABLE broker.broker_jobs ADD COLUMN IF NOT EXISTS job_group TEXT;
|
||||
UPDATE broker.broker_jobs SET job_group = job_name WHERE job_group IS NULL;
|
||||
ALTER TABLE broker.broker_jobs ALTER COLUMN job_group SET NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_broker_jobs_group
|
||||
ON broker.broker_jobs (tenant_id, job_group, complete_status);
|
||||
|
||||
ALTER TABLE broker.broker_job_dependency DROP CONSTRAINT IF EXISTS broker_job_dependency_pkey;
|
||||
ALTER TABLE broker.broker_job_dependency ALTER COLUMN depends_on_job_id DROP NOT NULL;
|
||||
ALTER TABLE broker.broker_job_dependency ADD COLUMN IF NOT EXISTS depends_on_group TEXT;
|
||||
ALTER TABLE broker.broker_job_dependency ADD COLUMN IF NOT EXISTS id_broker_job_dependency BIGSERIAL;
|
||||
ALTER TABLE broker.broker_job_dependency ADD PRIMARY KEY (id_broker_job_dependency);
|
||||
|
||||
ALTER TABLE broker.broker_job_dependency DROP CONSTRAINT IF EXISTS broker_job_dependency_one_target;
|
||||
ALTER TABLE broker.broker_job_dependency ADD CONSTRAINT broker_job_dependency_one_target CHECK (
|
||||
(depends_on_job_id IS NOT NULL AND depends_on_group IS NULL) OR
|
||||
(depends_on_job_id IS NULL AND depends_on_group IS NOT NULL)
|
||||
);
|
||||
|
||||
-- Plain (non-partial) unique constraint, matching the old PK's guarantee --
|
||||
-- NULLs in depends_on_job_id (the group-dependency rows) are never
|
||||
-- considered equal by a standard unique constraint, so this only constrains
|
||||
-- id-based rows, and keeps "ON CONFLICT (job_id, depends_on_job_id)" (no
|
||||
-- predicate needed) working for existing callers.
|
||||
ALTER TABLE broker.broker_job_dependency DROP CONSTRAINT IF EXISTS broker_job_dependency_unique_id;
|
||||
ALTER TABLE broker.broker_job_dependency ADD CONSTRAINT broker_job_dependency_unique_id
|
||||
UNIQUE (job_id, depends_on_job_id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_broker_job_dependency_unique_group
|
||||
ON broker.broker_job_dependency (job_id, depends_on_group) WHERE depends_on_group IS NOT NULL;
|
||||
|
||||
COMMENT ON COLUMN broker.broker_jobs.job_group IS 'Group tag for this job; defaults to job_name. Other jobs can depend on the whole group.';
|
||||
COMMENT ON COLUMN broker.broker_job_dependency.depends_on_group IS 'Alternative to depends_on_job_id: job_id is not claimable until every job with job_group = depends_on_group has completed';
|
||||
@@ -0,0 +1,184 @@
|
||||
-- broker.broker_add_job: adds job groups.
|
||||
-- p_job_group defaults to p_job_name when not given. p_depends_on_groups is
|
||||
-- the group-based counterpart to the existing p_depends_on_job_ids: the new
|
||||
-- job is not claimable until every job tagged with each named group has
|
||||
-- completed. Both dependency kinds can be combined on the same job.
|
||||
-- New parameters are appended after existing ones with defaults, so every
|
||||
-- existing positional call (however many args it passes) keeps working
|
||||
-- unchanged. Appending arguments changes the function's identity though --
|
||||
-- CREATE OR REPLACE would create a second, ambiguous overload rather than
|
||||
-- replacing -- so the old 10-arg signature is dropped explicitly first.
|
||||
|
||||
DROP FUNCTION IF EXISTS broker.broker_add_job(
|
||||
TEXT, TEXT, INTEGER, INTEGER, TEXT, TEXT, BIGINT, BIGINT[], TEXT, INTEGER
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_add_job(
|
||||
p_job_name TEXT,
|
||||
p_execute_str TEXT,
|
||||
p_job_queue INTEGER DEFAULT 1,
|
||||
p_job_priority INTEGER DEFAULT 0,
|
||||
p_job_language TEXT DEFAULT 'sql',
|
||||
p_run_as TEXT DEFAULT NULL,
|
||||
p_schedule_id BIGINT DEFAULT NULL,
|
||||
p_depends_on_job_ids BIGINT[] DEFAULT NULL,
|
||||
p_idempotency_key TEXT DEFAULT NULL,
|
||||
p_max_attempts INTEGER DEFAULT 1,
|
||||
p_job_group TEXT DEFAULT NULL,
|
||||
p_depends_on_groups TEXT[] DEFAULT NULL,
|
||||
OUT p_retval INTEGER,
|
||||
OUT p_errmsg TEXT,
|
||||
OUT p_job_id BIGINT
|
||||
)
|
||||
RETURNS RECORD
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
v_notification_payload JSON;
|
||||
v_tenant_id TEXT;
|
||||
v_job_group TEXT;
|
||||
v_dep_id BIGINT;
|
||||
v_dep_group TEXT;
|
||||
v_cycle_exists BOOLEAN;
|
||||
BEGIN
|
||||
p_retval := 0;
|
||||
p_errmsg := '';
|
||||
p_job_id := NULL;
|
||||
|
||||
IF p_job_name IS NULL OR p_job_name = '' THEN
|
||||
p_retval := 1;
|
||||
p_errmsg := 'Job name is required';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF p_execute_str IS NULL OR p_execute_str = '' THEN
|
||||
p_retval := 2;
|
||||
p_errmsg := 'Execute string is required';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF p_job_queue IS NULL OR p_job_queue <= 0 THEN
|
||||
p_retval := 3;
|
||||
p_errmsg := 'Invalid job queue number';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF p_max_attempts IS NULL OR p_max_attempts <= 0 THEN
|
||||
p_max_attempts := 1;
|
||||
END IF;
|
||||
|
||||
v_job_group := COALESCE(NULLIF(p_job_group, ''), p_job_name);
|
||||
|
||||
-- Falls back to 'default' when the caller never called broker_set_tenant,
|
||||
-- so single-tenant use (and the RLS WITH CHECK on insert) keeps working.
|
||||
v_tenant_id := COALESCE(NULLIF(current_setting('broker.tenant_id', true), ''), 'default');
|
||||
|
||||
IF p_idempotency_key IS NOT NULL THEN
|
||||
SELECT id_broker_jobs INTO p_job_id
|
||||
FROM broker.broker_jobs
|
||||
WHERE job_queue = p_job_queue
|
||||
AND idempotency_key = p_idempotency_key
|
||||
AND tenant_id = v_tenant_id;
|
||||
|
||||
IF FOUND THEN
|
||||
p_errmsg := 'Job with this idempotency key already exists; returning existing job id';
|
||||
RETURN;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
INSERT INTO broker.broker_jobs (
|
||||
job_name, job_group, job_priority, job_queue, job_language, execute_str, run_as,
|
||||
rid_broker_schedule, tenant_id, max_attempts, idempotency_key, complete_status
|
||||
) VALUES (
|
||||
p_job_name, v_job_group, p_job_priority, p_job_queue, p_job_language, p_execute_str, p_run_as,
|
||||
p_schedule_id, v_tenant_id, p_max_attempts, p_idempotency_key, 0
|
||||
)
|
||||
RETURNING id_broker_jobs INTO p_job_id;
|
||||
|
||||
IF p_depends_on_job_ids IS NOT NULL THEN
|
||||
FOREACH v_dep_id IN ARRAY p_depends_on_job_ids LOOP
|
||||
IF v_dep_id IS NULL THEN
|
||||
CONTINUE;
|
||||
END IF;
|
||||
|
||||
IF v_dep_id = p_job_id THEN
|
||||
p_retval := 20;
|
||||
p_errmsg := 'Invalid dependency: a job cannot depend on itself';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM broker.broker_job_dependency
|
||||
WHERE job_id = v_dep_id AND depends_on_job_id = p_job_id
|
||||
) INTO v_cycle_exists;
|
||||
|
||||
IF v_cycle_exists THEN
|
||||
p_retval := 21;
|
||||
p_errmsg := format('Invalid dependency: job %s already depends on %s (would create a cycle)', v_dep_id, p_job_id);
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
INSERT INTO broker.broker_job_dependency (job_id, depends_on_job_id, tenant_id)
|
||||
VALUES (p_job_id, v_dep_id, v_tenant_id)
|
||||
ON CONFLICT (job_id, depends_on_job_id) DO NOTHING;
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
IF p_depends_on_groups IS NOT NULL THEN
|
||||
FOREACH v_dep_group IN ARRAY p_depends_on_groups LOOP
|
||||
IF v_dep_group IS NULL OR v_dep_group = '' THEN
|
||||
CONTINUE;
|
||||
END IF;
|
||||
|
||||
IF v_dep_group = v_job_group THEN
|
||||
p_retval := 22;
|
||||
p_errmsg := format('Invalid dependency: a job cannot depend on its own group (%s)', v_job_group);
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM broker.broker_jobs j
|
||||
JOIN broker.broker_job_dependency d ON d.job_id = j.id_broker_jobs
|
||||
WHERE j.tenant_id = v_tenant_id
|
||||
AND j.job_group = v_dep_group
|
||||
AND d.depends_on_group = v_job_group
|
||||
) INTO v_cycle_exists;
|
||||
|
||||
IF v_cycle_exists THEN
|
||||
p_retval := 23;
|
||||
p_errmsg := format('Invalid dependency: group %s already depends on %s (would create a cycle)', v_dep_group, v_job_group);
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
INSERT INTO broker.broker_job_dependency (job_id, depends_on_group, tenant_id)
|
||||
VALUES (p_job_id, v_dep_group, v_tenant_id)
|
||||
ON CONFLICT (job_id, depends_on_group) WHERE depends_on_group IS NOT NULL DO NOTHING;
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
v_notification_payload := json_build_object(
|
||||
'queue', p_job_queue,
|
||||
'job_id', p_job_id
|
||||
);
|
||||
|
||||
PERFORM pg_notify('broker.event', v_notification_payload::text);
|
||||
|
||||
EXCEPTION
|
||||
WHEN unique_violation THEN
|
||||
-- Concurrent insert raced us to the same idempotency key.
|
||||
p_retval := 0;
|
||||
p_errmsg := 'Job with this idempotency key already exists; returning existing job id';
|
||||
SELECT id_broker_jobs INTO p_job_id
|
||||
FROM broker.broker_jobs
|
||||
WHERE job_queue = p_job_queue
|
||||
AND idempotency_key = p_idempotency_key
|
||||
AND tenant_id = v_tenant_id;
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 99;
|
||||
p_errmsg := SQLERRM;
|
||||
RAISE WARNING 'broker_add_job error: %', SQLERRM;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION broker.broker_add_job IS 'Adds a job (with optional id/group dependencies, job group, and idempotency key) and sends a wake-only NOTIFY';
|
||||
@@ -0,0 +1,91 @@
|
||||
-- broker.broker_get: also gates claiming on group-based dependencies
|
||||
-- (broker_job_dependency.depends_on_group) alongside the existing id-based
|
||||
-- ones. Signature is unchanged, only the eligibility query grows a second
|
||||
-- NOT EXISTS clause.
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_get(
|
||||
p_queue_number INTEGER,
|
||||
p_instance_id BIGINT DEFAULT NULL,
|
||||
p_lease_seconds INTEGER DEFAULT 60,
|
||||
OUT p_retval INTEGER,
|
||||
OUT p_errmsg TEXT,
|
||||
OUT p_job_id BIGINT,
|
||||
OUT p_lease_token UUID
|
||||
)
|
||||
RETURNS RECORD
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
v_job_id BIGINT;
|
||||
v_lease_token UUID;
|
||||
BEGIN
|
||||
p_retval := 0;
|
||||
p_errmsg := '';
|
||||
p_job_id := NULL;
|
||||
p_lease_token := NULL;
|
||||
|
||||
IF p_queue_number IS NULL OR p_queue_number <= 0 THEN
|
||||
p_retval := 1;
|
||||
p_errmsg := 'Invalid queue number';
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF p_lease_seconds IS NULL OR p_lease_seconds <= 0 THEN
|
||||
p_lease_seconds := 60;
|
||||
END IF;
|
||||
|
||||
SELECT candidate.id_broker_jobs
|
||||
INTO v_job_id
|
||||
FROM broker.broker_jobs candidate
|
||||
WHERE candidate.job_queue = p_queue_number
|
||||
AND candidate.complete_status = 0
|
||||
AND candidate.available_at <= NOW()
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM broker.broker_job_dependency d
|
||||
JOIN broker.broker_jobs dep ON dep.id_broker_jobs = d.depends_on_job_id
|
||||
WHERE d.job_id = candidate.id_broker_jobs
|
||||
AND dep.complete_status <> 2
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM broker.broker_job_dependency d
|
||||
JOIN broker.broker_jobs dep ON dep.tenant_id = candidate.tenant_id
|
||||
AND dep.job_group = d.depends_on_group
|
||||
WHERE d.job_id = candidate.id_broker_jobs
|
||||
AND dep.id_broker_jobs <> candidate.id_broker_jobs
|
||||
AND dep.complete_status <> 2
|
||||
)
|
||||
ORDER BY candidate.job_priority DESC, candidate.created_at ASC, candidate.id_broker_jobs ASC
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
v_lease_token := gen_random_uuid();
|
||||
|
||||
UPDATE broker.broker_jobs
|
||||
SET complete_status = 1, -- running
|
||||
started_at = NOW(),
|
||||
rid_broker_queueinstance = p_instance_id,
|
||||
attempt_count = attempt_count + 1,
|
||||
lease_token = v_lease_token,
|
||||
leased_at = NOW(),
|
||||
lease_expires_at = NOW() + make_interval(secs => p_lease_seconds),
|
||||
updated_at = NOW()
|
||||
WHERE id_broker_jobs = v_job_id;
|
||||
|
||||
p_job_id := v_job_id;
|
||||
p_lease_token := v_lease_token;
|
||||
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
p_retval := 2;
|
||||
p_errmsg := SQLERRM;
|
||||
RAISE WARNING 'broker_get error: %', SQLERRM;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION broker.broker_get IS 'Claims the next eligible job from a queue (honoring id- and group-based dependencies) and grants a lease';
|
||||
@@ -0,0 +1,41 @@
|
||||
-- broker.broker_add_job_simple
|
||||
-- Convenience wrapper around broker.broker_add_job for the common case: a
|
||||
-- job named p_job_name running p_execute_str at p_job_priority, depending on
|
||||
-- other jobs by group name (p_depends_on_groups) -- every job's group
|
||||
-- defaults to its own job_name, so passing job names here just works.
|
||||
-- Pure pass-through: no name-to-id resolution needed since dependencies are
|
||||
-- resolved live, by group, inside broker_get.
|
||||
|
||||
CREATE OR REPLACE FUNCTION broker.broker_add_job_simple(
|
||||
p_job_name TEXT,
|
||||
p_execute_str TEXT,
|
||||
p_job_priority INTEGER DEFAULT 0,
|
||||
p_depends_on_groups TEXT[] DEFAULT NULL,
|
||||
OUT p_retval INTEGER,
|
||||
OUT p_errmsg TEXT,
|
||||
OUT p_job_id BIGINT
|
||||
)
|
||||
RETURNS RECORD
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
SELECT r.p_retval, r.p_errmsg, r.p_job_id
|
||||
INTO p_retval, p_errmsg, p_job_id
|
||||
FROM broker.broker_add_job(
|
||||
p_job_name,
|
||||
p_execute_str,
|
||||
1, -- p_job_queue
|
||||
p_job_priority,
|
||||
'sql', -- p_job_language
|
||||
NULL, -- p_run_as
|
||||
NULL, -- p_schedule_id
|
||||
NULL, -- p_depends_on_job_ids
|
||||
NULL, -- p_idempotency_key
|
||||
1, -- p_max_attempts
|
||||
NULL, -- p_job_group (defaults to p_job_name)
|
||||
p_depends_on_groups
|
||||
) AS r;
|
||||
END;
|
||||
$$;
|
||||
|
||||
COMMENT ON FUNCTION broker.broker_add_job_simple IS 'Shortcut for broker_add_job: name, execute string, priority, and dependencies by group name (defaults to job name)';
|
||||
Reference in New Issue
Block a user