wdevs/pgsql-broker
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 🎉 postgresql broker first commit of forked prototype from my original code

Browse Source
This commit is contained in:
Warky
2026-01-02 20:56:39 +02:00
parent e90e5902cd
commit 19e469ff54
29 changed files with 3325 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
pkg/broker/install/sql/procedures/00_install.sql Normal file
View File

@@ -0,0 +1,13 @@
-- PostgreSQL Broker Procedures Installation Script
-- Run this script to create all required stored procedures
\echo 'Installing PostgreSQL Broker procedures...'
\i 01_broker_get.sql
\i 02_broker_run.sql
\i 03_broker_set.sql
\i 04_broker_register_instance.sql
\i 05_broker_add_job.sql
\i 06_broker_ping_instance.sql
\echo 'PostgreSQL Broker procedures installed successfully!'

76
pkg/broker/install/sql/procedures/01_broker_get.sql Normal file
View File

@@ -0,0 +1,76 @@
-- broker_get function
-- Fetches the next job from the queue for a given queue number
-- Returns: p_retval (0=success, >0=error), p_errmsg (error message), p_job_id (job ID if found)
CREATE OR REPLACE FUNCTION broker_get(
p_queue_number INTEGER,
p_instance_id BIGINT DEFAULT NULL,
OUT p_retval INTEGER,
OUT p_errmsg TEXT,
OUT p_job_id BIGINT
)
RETURNS RECORD
LANGUAGE plpgsql
AS $$
DECLARE
v_job_record RECORD;
BEGIN
p_retval := 0;
p_errmsg := '';
p_job_id := NULL;
-- Validate queue number
IF p_queue_number IS NULL OR p_queue_number <= 0 THEN
p_retval := 1;
p_errmsg := 'Invalid queue number';
RETURN;
END IF;
-- Find and lock the next pending job for this queue
-- Uses SKIP LOCKED to avoid blocking on jobs being processed by other workers
-- Skip jobs with pending dependencies
SELECT id_broker_jobs, job_name, job_priority, execute_str
INTO v_job_record
FROM broker_jobs
WHERE job_queue = p_queue_number
AND complete_status = 0 -- pending
AND (
depends_on IS NULL -- no dependencies
OR depends_on = '{}' -- empty dependencies
OR NOT EXISTS ( -- all dependencies completed
SELECT 1
FROM broker_jobs dep
WHERE dep.job_name = ANY(broker_jobs.depends_on)
AND dep.complete_status = 0 -- pending dependency
)
)
ORDER BY job_priority DESC, created_at ASC
LIMIT 1
FOR UPDATE SKIP LOCKED;
-- If no job found, return success with NULL job_id
IF NOT FOUND THEN
RETURN;
END IF;
-- Update job status to running
UPDATE broker_jobs
SET complete_status = 1, -- running
started_at = NOW(),
rid_broker_queueinstance = p_instance_id,
updated_at = NOW()
WHERE id_broker_jobs = v_job_record.id_broker_jobs;
-- Return the job ID
p_job_id := v_job_record.id_broker_jobs;
EXCEPTION
WHEN OTHERS THEN
p_retval := 2;
p_errmsg := SQLERRM;
RAISE WARNING 'broker_get error: %', SQLERRM;
END;
$$;
-- Comments
COMMENT ON FUNCTION broker_get IS 'Fetches the next pending job from the specified queue';

113
pkg/broker/install/sql/procedures/02_broker_run.sql Normal file
View File

@@ -0,0 +1,113 @@
-- broker_run function
-- Executes a job by its ID
-- Returns: p_retval (0=success, >0=error), p_errmsg (error message)
CREATE OR REPLACE FUNCTION broker_run(
p_job_id BIGINT,
OUT p_retval INTEGER,
OUT p_errmsg TEXT
)
RETURNS RECORD
LANGUAGE plpgsql
AS $$
DECLARE
v_job_record RECORD;
v_execute_result TEXT;
v_error_occurred BOOLEAN := false;
BEGIN
p_retval := 0;
p_errmsg := '';
v_execute_result := '';
-- Validate job ID
IF p_job_id IS NULL OR p_job_id <= 0 THEN
p_retval := 1;
p_errmsg := 'Invalid job ID';
RETURN;
END IF;
-- Get job details
SELECT id_broker_jobs, execute_str, job_language, run_as, complete_status
INTO v_job_record
FROM broker_jobs
WHERE id_broker_jobs = p_job_id
FOR UPDATE;
-- Check if job exists
IF NOT FOUND THEN
p_retval := 2;
p_errmsg := 'Job not found';
RETURN;
END IF;
-- Check if job is in running state
IF v_job_record.complete_status != 1 THEN
p_retval := 3;
p_errmsg := format('Job is not in running state (status: %s)', v_job_record.complete_status);
RETURN;
END IF;
-- Execute the job
BEGIN
-- For SQL/PLPGSQL jobs, execute directly
IF v_job_record.job_language IN ('sql', 'plpgsql') THEN
EXECUTE v_job_record.execute_str;
v_execute_result := 'Success';
ELSE
-- Other languages would need external execution
p_retval := 4;
p_errmsg := format('Unsupported job language: %s', v_job_record.job_language);
v_error_occurred := true;
END IF;
EXCEPTION
WHEN OTHERS THEN
v_error_occurred := true;
p_retval := 5;
p_errmsg := SQLERRM;
v_execute_result := format('Error: %s', SQLERRM);
END;
-- Update job with results
IF v_error_occurred THEN
UPDATE broker_jobs
SET complete_status = 3, -- failed
error_msg = p_errmsg,
execute_result = v_execute_result,
completed_at = NOW(),
updated_at = NOW()
WHERE id_broker_jobs = p_job_id;
ELSE
UPDATE broker_jobs
SET complete_status = 2, -- completed
execute_result = v_execute_result,
error_msg = NULL,
completed_at = NOW(),
updated_at = NOW()
WHERE id_broker_jobs = p_job_id;
END IF;
EXCEPTION
WHEN OTHERS THEN
p_retval := 6;
p_errmsg := SQLERRM;
RAISE WARNING 'broker_run error: %', SQLERRM;
-- Try to update job status to failed
BEGIN
UPDATE broker_jobs
SET complete_status = 3, -- failed
error_msg = SQLERRM,
completed_at = NOW(),
updated_at = NOW()
WHERE id_broker_jobs = p_job_id;
EXCEPTION
WHEN OTHERS THEN
-- Ignore update errors
NULL;
END;
END;
$$;
-- Comments
COMMENT ON FUNCTION broker_run IS 'Executes a job by its ID and updates the status';

95
pkg/broker/install/sql/procedures/03_broker_set.sql Normal file
View File

@@ -0,0 +1,95 @@
-- broker_set function
-- Sets broker runtime options and context
-- Supports: user, application_name, and custom settings
-- Returns: p_retval (0=success, >0=error), p_errmsg (error message)
CREATE OR REPLACE FUNCTION broker_set(
p_option_name TEXT,
p_option_value TEXT,
OUT p_retval INTEGER,
OUT p_errmsg TEXT
)
RETURNS RECORD
LANGUAGE plpgsql
AS $$
DECLARE
v_sql TEXT;
BEGIN
p_retval := 0;
p_errmsg := '';
-- Validate inputs
IF p_option_name IS NULL OR p_option_name = '' THEN
p_retval := 1;
p_errmsg := 'Option name is required';
RETURN;
END IF;
-- Handle different option types
CASE LOWER(p_option_name)
WHEN 'user' THEN
-- Set session user context
-- This is useful for audit trails and permissions
BEGIN
v_sql := format('SET SESSION AUTHORIZATION %I', p_option_value);
EXECUTE v_sql;
EXCEPTION
WHEN OTHERS THEN
p_retval := 2;
p_errmsg := format('Failed to set user: %s', SQLERRM);
RETURN;
END;
WHEN 'application_name' THEN
-- Set application name (visible in pg_stat_activity)
BEGIN
v_sql := format('SET application_name TO %L', p_option_value);
EXECUTE v_sql;
EXCEPTION
WHEN OTHERS THEN
p_retval := 3;
p_errmsg := format('Failed to set application_name: %s', SQLERRM);
RETURN;
END;
WHEN 'search_path' THEN
-- Set schema search path
BEGIN
v_sql := format('SET search_path TO %s', p_option_value);
EXECUTE v_sql;
EXCEPTION
WHEN OTHERS THEN
p_retval := 4;
p_errmsg := format('Failed to set search_path: %s', SQLERRM);
RETURN;
END;
WHEN 'timezone' THEN
-- Set timezone
BEGIN
v_sql := format('SET timezone TO %L', p_option_value);
EXECUTE v_sql;
EXCEPTION
WHEN OTHERS THEN
p_retval := 5;
p_errmsg := format('Failed to set timezone: %s', SQLERRM);
RETURN;
END;
ELSE
-- Unknown option
p_retval := 10;
p_errmsg := format('Unknown option: %s', p_option_name);
RETURN;
END CASE;
EXCEPTION
WHEN OTHERS THEN
p_retval := 99;
p_errmsg := SQLERRM;
RAISE WARNING 'broker_set error: %', SQLERRM;
END;
$$;
-- Comments
COMMENT ON FUNCTION broker_set IS 'Sets broker runtime options and session context (user, application_name, search_path, timezone)';

82
pkg/broker/install/sql/procedures/04_broker_register_instance.sql Normal file
View File

@@ -0,0 +1,82 @@
-- broker_register_instance function
-- Registers a new broker instance in the database
-- Returns: p_retval (0=success, >0=error), p_errmsg (error message), p_instance_id (new instance ID)
CREATE OR REPLACE FUNCTION broker_register_instance(
p_name TEXT,
p_hostname TEXT,
p_pid INTEGER,
p_version TEXT,
p_queue_count INTEGER,
OUT p_retval INTEGER,
OUT p_errmsg TEXT,
OUT p_instance_id BIGINT
)
RETURNS RECORD
LANGUAGE plpgsql
AS $$
DECLARE
v_active_count INTEGER;
BEGIN
p_retval := 0;
p_errmsg := '';
p_instance_id := NULL;
-- Validate inputs
IF p_name IS NULL OR p_name = '' THEN
p_retval := 1;
p_errmsg := 'Instance name is required';
RETURN;
END IF;
IF p_hostname IS NULL OR p_hostname = '' THEN
p_retval := 2;
p_errmsg := 'Hostname is required';
RETURN;
END IF;
-- Check for existing active instances
-- Only one broker instance should be active per database
SELECT COUNT(*)
INTO v_active_count
FROM broker_queueinstance
WHERE status = 'active';
IF v_active_count > 0 THEN
p_retval := 3;
p_errmsg := 'Another broker instance is already active in this database. Only one broker instance per database is allowed.';
RETURN;
END IF;
-- Insert new instance
INSERT INTO broker_queueinstance (
name,
hostname,
pid,
version,
status,
queue_count,
started_at,
last_ping_at
) VALUES (
p_name,
p_hostname,
p_pid,
p_version,
'active',
p_queue_count,
NOW(),
NOW()
)
RETURNING id_broker_queueinstance INTO p_instance_id;
EXCEPTION
WHEN OTHERS THEN
p_retval := 99;
p_errmsg := SQLERRM;
RAISE WARNING 'broker_register_instance error: %', SQLERRM;
END;
$$;
-- Comments
COMMENT ON FUNCTION broker_register_instance IS 'Registers a new broker instance';

91
pkg/broker/install/sql/procedures/05_broker_add_job.sql Normal file
View File

@@ -0,0 +1,91 @@
-- broker_add_job function
-- Adds a new job to the broker queue and sends a notification
-- Returns: p_retval (0=success, >0=error), p_errmsg (error message), p_job_id (new job ID)
CREATE OR REPLACE FUNCTION broker_add_job(
p_job_name TEXT,
p_execute_str TEXT,
p_job_queue INTEGER DEFAULT 1,
p_job_priority INTEGER DEFAULT 0,
p_job_language TEXT DEFAULT 'sql',
p_run_as TEXT DEFAULT NULL,
p_schedule_id BIGINT DEFAULT NULL,
p_depends_on TEXT[] DEFAULT NULL,
OUT p_retval INTEGER,
OUT p_errmsg TEXT,
OUT p_job_id BIGINT
)
RETURNS RECORD
LANGUAGE plpgsql
AS $$
DECLARE
v_notification_payload JSON;
BEGIN
p_retval := 0;
p_errmsg := '';
p_job_id := NULL;
-- Validate inputs
IF p_job_name IS NULL OR p_job_name = '' THEN
p_retval := 1;
p_errmsg := 'Job name is required';
RETURN;
END IF;
IF p_execute_str IS NULL OR p_execute_str = '' THEN
p_retval := 2;
p_errmsg := 'Execute string is required';
RETURN;
END IF;
IF p_job_queue IS NULL OR p_job_queue <= 0 THEN
p_retval := 3;
p_errmsg := 'Invalid job queue number';
RETURN;
END IF;
-- Insert new job
INSERT INTO broker_jobs (
job_name,
job_priority,
job_queue,
job_language,
execute_str,
run_as,
rid_broker_schedule,
depends_on,
complete_status
) VALUES (
p_job_name,
p_job_priority,
p_job_queue,
p_job_language,
p_execute_str,
p_run_as,
p_schedule_id,
p_depends_on,
0 -- pending
)
RETURNING id_broker_jobs INTO p_job_id;
-- Create notification payload
v_notification_payload := json_build_object(
'id', p_job_id,
'job_name', p_job_name,
'job_queue', p_job_queue,
'job_priority', p_job_priority
);
-- Send notification to broker
PERFORM pg_notify('broker.event', v_notification_payload::text);
EXCEPTION
WHEN OTHERS THEN
p_retval := 99;
p_errmsg := SQLERRM;
RAISE WARNING 'broker_add_job error: %', SQLERRM;
END;
$$;
-- Comments
COMMENT ON FUNCTION broker_add_job IS 'Adds a new job to the broker queue and sends a NOTIFY event';

98
pkg/broker/install/sql/procedures/06_broker_ping_instance.sql Normal file
View File

@@ -0,0 +1,98 @@
-- broker_ping_instance function
-- Updates the last_ping_at timestamp for a broker instance
-- Returns: p_retval (0=success, >0=error), p_errmsg (error message)
CREATE OR REPLACE FUNCTION broker_ping_instance(
p_instance_id BIGINT,
p_jobs_handled BIGINT DEFAULT NULL,
OUT p_retval INTEGER,
OUT p_errmsg TEXT
)
RETURNS RECORD
LANGUAGE plpgsql
AS $$
BEGIN
p_retval := 0;
p_errmsg := '';
-- Validate instance ID
IF p_instance_id IS NULL OR p_instance_id <= 0 THEN
p_retval := 1;
p_errmsg := 'Invalid instance ID';
RETURN;
END IF;
-- Update ping timestamp
IF p_jobs_handled IS NOT NULL THEN
UPDATE broker_queueinstance
SET last_ping_at = NOW(),
jobs_handled = p_jobs_handled
WHERE id_broker_queueinstance = p_instance_id;
ELSE
UPDATE broker_queueinstance
SET last_ping_at = NOW()
WHERE id_broker_queueinstance = p_instance_id;
END IF;
-- Check if instance was found
IF NOT FOUND THEN
p_retval := 2;
p_errmsg := 'Instance not found';
RETURN;
END IF;
EXCEPTION
WHEN OTHERS THEN
p_retval := 99;
p_errmsg := SQLERRM;
RAISE WARNING 'broker_ping_instance error: %', SQLERRM;
END;
$$;
-- broker_shutdown_instance function
-- Marks a broker instance as shutdown
-- Returns: p_retval (0=success, >0=error), p_errmsg (error message)
CREATE OR REPLACE FUNCTION broker_shutdown_instance(
p_instance_id BIGINT,
OUT p_retval INTEGER,
OUT p_errmsg TEXT
)
RETURNS RECORD
LANGUAGE plpgsql
AS $$
BEGIN
p_retval := 0;
p_errmsg := '';
-- Validate instance ID
IF p_instance_id IS NULL OR p_instance_id <= 0 THEN
p_retval := 1;
p_errmsg := 'Invalid instance ID';
RETURN;
END IF;
-- Update instance status
UPDATE broker_queueinstance
SET status = 'shutdown',
shutdown_at = NOW()
WHERE id_broker_queueinstance = p_instance_id;
-- Check if instance was found
IF NOT FOUND THEN
p_retval := 2;
p_errmsg := 'Instance not found';
RETURN;
END IF;
EXCEPTION
WHEN OTHERS THEN
p_retval := 99;
p_errmsg := SQLERRM;
RAISE WARNING 'broker_shutdown_instance error: %', SQLERRM;
END;
$$;
-- Comments
COMMENT ON FUNCTION broker_ping_instance IS 'Updates the last ping timestamp for an instance';
COMMENT ON FUNCTION broker_shutdown_instance IS 'Marks an instance as shutdown';