Compare commits

...

13 Commits
v1.0.3 ... main

Author SHA1 Message Date
037bd4c05e chore: 🦺 actions updated
Some checks failed
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -25m45s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -25m23s
Build , Vet Test, and Lint / Lint Code (push) Successful in -25m16s
Build , Vet Test, and Lint / Build (push) Successful in -25m28s
Tests / Unit Tests (push) Successful in -25m45s
Tests / Integration Tests (push) Failing after -25m51s
2025-12-31 19:23:09 +02:00
Hein
e77468a239 refactor: ♻️ ach package accepts its configuration as a parameter rather than reading from global config
Some checks failed
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -25m44s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -25m19s
Build , Vet Test, and Lint / Lint Code (push) Successful in -25m18s
Build , Vet Test, and Lint / Build (push) Successful in -25m30s
Tests / Unit Tests (push) Failing after -25m51s
Tests / Integration Tests (push) Failing after -25m53s
2025-12-31 12:39:59 +02:00
Hein
82d84435f2 Fixed version release auto inc
Some checks failed
Build , Vet Test, and Lint / Run Vet Tests (1.24.x) (push) Successful in -25m36s
Build , Vet Test, and Lint / Run Vet Tests (1.23.x) (push) Successful in -25m5s
Build , Vet Test, and Lint / Lint Code (push) Successful in -25m16s
Build , Vet Test, and Lint / Build (push) Successful in -25m29s
Tests / Unit Tests (push) Failing after -25m48s
Tests / Integration Tests (push) Failing after -25m53s
2025-12-31 09:41:23 +02:00
Bitech Systems
b99b08430e Merge pull request #16 from bitechdev/fix-funcapi
fix: 🔒 Fixed funcapi possible injections
2025-12-31 09:38:44 +02:00
Hein
fae9a082bd refactor: 🚨 linting issues 2025-12-31 09:37:12 +02:00
Hein Puth (Warkanum)
191822b91c Merge pull request #17 from bitechdev/copilot/sub-pr-16
fix: Comprehensive SQL injection protection in funcspec handlers
2025-12-31 09:35:49 +02:00
copilot-swe-agent[bot]
a6a17d019f fix: Optimized regex performance and added backslash escaping for LIKE patterns
- Added backslash escaping to LIKE pattern sanitization
- Optimized ValidSQL select mode with single pre-compiled regex
- All funcspec tests pass (except pre-existing TestReplaceMetaVariables)

Co-authored-by: warkanum <208308+warkanum@users.noreply.github.com>
2025-12-31 07:28:16 +00:00
copilot-swe-agent[bot]
a7cc42044b fix: Improved SQL injection protection with proper handling
- Fixed IN clause to conditionally quote only string values (not numeric)
- Fixed LIKE pattern sanitization to preserve wildcards while preventing injection
- Improved dangerous pattern removal with case-insensitive regex while preserving case
- All funcspec tests now pass (except pre-existing TestReplaceMetaVariables)

Co-authored-by: warkanum <208308+warkanum@users.noreply.github.com>
2025-12-31 07:24:56 +00:00
copilot-swe-agent[bot]
8cdc353029 fix: Fixed SQL quoting for empty/zero values and sanitized match filter
- Sanitize colval immediately after extraction in match= filter
- Fixed empty/zero value handling to use proper SQL literals (0 vs '')
- Applied proper quoting for string vs numeric comparisons
- Fixed x-fieldfilter handlers for proper value handling

Co-authored-by: warkanum <208308+warkanum@users.noreply.github.com>
2025-12-31 07:22:42 +00:00
copilot-swe-agent[bot]
6528e94297 fix: Improved SQL injection protections based on code review
- Fixed backslash escaping order in colvalue mode
- Added proper quoting for IN clause values
- Simplified dangerous pattern matching with case-insensitive approach
- All funcspec tests pass (except pre-existing test failure)

Co-authored-by: warkanum <208308+warkanum@users.noreply.github.com>
2025-12-31 07:20:56 +00:00
copilot-swe-agent[bot]
f711bf38d2 fix: Enhanced SQL injection protection in funcspec
- Added sanitization for path parameters in mergePathParams
- Added sanitization for query parameters with p- prefix in mergeQueryParams
- Added sanitization for header parameters in mergeHeaderParams
- Fixed IN clause to sanitize all values individually
- Improved ValidSQL function with better escaping and more injection patterns
- Added backslash escaping to colvalue mode
- Extended dangerous keyword list in select mode

Co-authored-by: warkanum <208308+warkanum@users.noreply.github.com>
2025-12-31 07:19:53 +00:00
copilot-swe-agent[bot]
44356d8750 Initial plan 2025-12-31 07:11:57 +00:00
Hein
caf85cf558 fix: 🔒 Fixed funcapi possible injections 2025-12-31 09:09:16 +02:00
7 changed files with 375 additions and 54 deletions

View File

@@ -17,11 +17,13 @@ jobs:
- name: Run unit tests - name: Run unit tests
run: go test ./pkg/resolvespec ./pkg/restheadspec -v -cover run: go test ./pkg/resolvespec ./pkg/restheadspec -v -cover
- name: Generate coverage report - name: Generate coverage report
continue-on-error: true
run: | run: |
go test ./pkg/resolvespec ./pkg/restheadspec -coverprofile=coverage.out go test ./pkg/resolvespec ./pkg/restheadspec -coverprofile=coverage.out
go tool cover -html=coverage.out -o coverage.html go tool cover -html=coverage.out -o coverage.html
- name: Upload coverage - name: Upload coverage
uses: actions/upload-artifact@v5 uses: actions/upload-artifact@v5
continue-on-error: true
with: with:
name: coverage-report name: coverage-report
path: coverage.html path: coverage.html
@@ -55,27 +57,34 @@ jobs:
psql -h localhost -U postgres -c "CREATE DATABASE resolvespec_test;" psql -h localhost -U postgres -c "CREATE DATABASE resolvespec_test;"
psql -h localhost -U postgres -c "CREATE DATABASE restheadspec_test;" psql -h localhost -U postgres -c "CREATE DATABASE restheadspec_test;"
- name: Run resolvespec integration tests - name: Run resolvespec integration tests
continue-on-error: true
env: env:
TEST_DATABASE_URL: "host=localhost user=postgres password=postgres dbname=resolvespec_test port=5432 sslmode=disable" TEST_DATABASE_URL: "host=localhost user=postgres password=postgres dbname=resolvespec_test port=5432 sslmode=disable"
run: go test -tags=integration ./pkg/resolvespec -v -coverprofile=coverage-resolvespec-integration.out run: go test -tags=integration ./pkg/resolvespec -v -coverprofile=coverage-resolvespec-integration.out
- name: Run restheadspec integration tests - name: Run restheadspec integration tests
continue-on-error: true
env: env:
TEST_DATABASE_URL: "host=localhost user=postgres password=postgres dbname=restheadspec_test port=5432 sslmode=disable" TEST_DATABASE_URL: "host=localhost user=postgres password=postgres dbname=restheadspec_test port=5432 sslmode=disable"
run: go test -tags=integration ./pkg/restheadspec -v -coverprofile=coverage-restheadspec-integration.out run: go test -tags=integration ./pkg/restheadspec -v -coverprofile=coverage-restheadspec-integration.out
- name: Generate integration coverage - name: Generate integration coverage
continue-on-error: true
env: env:
TEST_DATABASE_URL: "host=localhost user=postgres password=postgres dbname=resolvespec_test port=5432 sslmode=disable" TEST_DATABASE_URL: "host=localhost user=postgres password=postgres dbname=resolvespec_test port=5432 sslmode=disable"
run: | run: |
go tool cover -html=coverage-resolvespec-integration.out -o coverage-resolvespec-integration.html go tool cover -html=coverage-resolvespec-integration.out -o coverage-resolvespec-integration.html
go tool cover -html=coverage-restheadspec-integration.out -o coverage-restheadspec-integration.html go tool cover -html=coverage-restheadspec-integration.out -o coverage-restheadspec-integration.html
- name: Upload resolvespec integration coverage - name: Upload resolvespec integration coverage
uses: actions/upload-artifact@v5 uses: actions/upload-artifact@v5
continue-on-error: true
with: with:
name: resolvespec-integration-coverage-report name: resolvespec-integration-coverage-report
path: coverage-resolvespec-integration.html path: coverage-resolvespec-integration.html
- name: Upload restheadspec integration coverage - name: Upload restheadspec integration coverage
uses: actions/upload-artifact@v5 uses: actions/upload-artifact@v5
continue-on-error: true
with: with:
name: integration-coverage-restheadspec-report name: integration-coverage-restheadspec-report
path: coverage-restheadspec-integration path: coverage-restheadspec-integration

View File

@@ -13,14 +13,22 @@ test-integration:
# Run all tests (unit + integration) # Run all tests (unit + integration)
test: test-unit test-integration test: test-unit test-integration
release-version: ## Create and push a release with specific version (use: make release-version VERSION=v1.2.3) release-version: ## Create and push a release with specific version (use: make release-version VERSION=v1.2.3 or make release-version to auto-increment)
@if [ -z "$(VERSION)" ]; then \ @if [ -z "$(VERSION)" ]; then \
echo "Error: VERSION is required. Usage: make release-version VERSION=v1.2.3"; \ latest_tag=$$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0"); \
exit 1; \ echo "No VERSION specified. Last version: $$latest_tag"; \
fi version_num=$$(echo "$$latest_tag" | sed 's/^v//'); \
@version="$(VERSION)"; \ major=$$(echo "$$version_num" | cut -d. -f1); \
if ! echo "$$version" | grep -q "^v"; then \ minor=$$(echo "$$version_num" | cut -d. -f2); \
version="v$$version"; \ patch=$$(echo "$$version_num" | cut -d. -f3); \
new_patch=$$((patch + 1)); \
version="v$$major.$$minor.$$new_patch"; \
echo "Auto-incrementing to: $$version"; \
else \
version="$(VERSION)"; \
if ! echo "$$version" | grep -q "^v"; then \
version="v$$version"; \
fi; \
fi; \ fi; \
echo "Creating release: $$version"; \ echo "Creating release: $$version"; \
latest_tag=$$(git describe --tags --abbrev=0 2>/dev/null || echo ""); \ latest_tag=$$(git describe --tags --abbrev=0 2>/dev/null || echo ""); \

View File

@@ -84,7 +84,7 @@ func (h *Handler) SqlQueryList(sqlquery string, options SqlQueryOptions) HTTPFun
// Create local copy to avoid modifying the captured parameter across requests // Create local copy to avoid modifying the captured parameter across requests
sqlquery := sqlquery sqlquery := sqlquery
ctx, cancel := context.WithTimeout(r.Context(), 900*time.Second) ctx, cancel := context.WithTimeout(r.Context(), 15*time.Minute)
defer cancel() defer cancel()
var dbobjlist []map[string]interface{} var dbobjlist []map[string]interface{}
@@ -423,7 +423,7 @@ func (h *Handler) SqlQuery(sqlquery string, options SqlQueryOptions) HTTPFuncTyp
// Create local copy to avoid modifying the captured parameter across requests // Create local copy to avoid modifying the captured parameter across requests
sqlquery := sqlquery sqlquery := sqlquery
ctx, cancel := context.WithTimeout(r.Context(), 600*time.Second) ctx, cancel := context.WithTimeout(r.Context(), 15*time.Minute)
defer cancel() defer cancel()
propQry := make(map[string]string) propQry := make(map[string]string)
@@ -522,10 +522,17 @@ func (h *Handler) SqlQuery(sqlquery string, options SqlQueryOptions) HTTPFuncTyp
if strings.HasPrefix(kLower, "x-fieldfilter-") { if strings.HasPrefix(kLower, "x-fieldfilter-") {
colname := strings.ReplaceAll(kLower, "x-fieldfilter-", "") colname := strings.ReplaceAll(kLower, "x-fieldfilter-", "")
if strings.Contains(strings.ToLower(sqlquery), colname) { if strings.Contains(strings.ToLower(sqlquery), colname) {
if val == "" || val == "0" { switch val {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("COALESCE(%s, 0) = %s", ValidSQL(colname, "colname"), ValidSQL(val, "colvalue"))) case "0":
} else { sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("COALESCE(%s, 0) = 0", ValidSQL(colname, "colname")))
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s = %s", ValidSQL(colname, "colname"), ValidSQL(val, "colvalue"))) case "":
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("(%[1]s = '' OR %[1]s IS NULL)", ValidSQL(colname, "colname")))
default:
if IsNumeric(val) {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s = %s", ValidSQL(colname, "colname"), ValidSQL(val, "colvalue")))
} else {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s = '%s'", ValidSQL(colname, "colname"), ValidSQL(val, "colvalue")))
}
} }
} }
} }
@@ -662,7 +669,10 @@ func (h *Handler) mergePathParams(r *http.Request, sqlquery string, variables ma
for k, v := range pathVars { for k, v := range pathVars {
kword := fmt.Sprintf("[%s]", k) kword := fmt.Sprintf("[%s]", k)
if strings.Contains(sqlquery, kword) { if strings.Contains(sqlquery, kword) {
sqlquery = strings.ReplaceAll(sqlquery, kword, fmt.Sprintf("%v", v)) // Sanitize the value before replacing
vStr := fmt.Sprintf("%v", v)
sanitized := ValidSQL(vStr, "colvalue")
sqlquery = strings.ReplaceAll(sqlquery, kword, sanitized)
} }
variables[k] = v variables[k] = v
@@ -690,7 +700,9 @@ func (h *Handler) mergeQueryParams(r *http.Request, sqlquery string, variables m
// Replace in SQL if placeholder exists // Replace in SQL if placeholder exists
if strings.Contains(sqlquery, kword) && len(val) > 0 { if strings.Contains(sqlquery, kword) && len(val) > 0 {
if strings.HasPrefix(parmk, "p-") { if strings.HasPrefix(parmk, "p-") {
sqlquery = strings.ReplaceAll(sqlquery, kword, val) // Sanitize the parameter value before replacing
sanitized := ValidSQL(val, "colvalue")
sqlquery = strings.ReplaceAll(sqlquery, kword, sanitized)
} }
} }
@@ -702,15 +714,36 @@ func (h *Handler) mergeQueryParams(r *http.Request, sqlquery string, variables m
// Apply filters if allowed // Apply filters if allowed
if allowFilter && len(parmk) > 1 && strings.Contains(strings.ToLower(sqlquery), strings.ToLower(parmk)) { if allowFilter && len(parmk) > 1 && strings.Contains(strings.ToLower(sqlquery), strings.ToLower(parmk)) {
if len(parmv) > 1 { if len(parmv) > 1 {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s IN (%s)", ValidSQL(parmk, "colname"), strings.Join(parmv, ","))) // Sanitize each value in the IN clause with appropriate quoting
sanitizedValues := make([]string, len(parmv))
for i, v := range parmv {
if IsNumeric(v) {
// Numeric values don't need quotes
sanitizedValues[i] = ValidSQL(v, "colvalue")
} else {
// String values need quotes
sanitized := ValidSQL(v, "colvalue")
sanitizedValues[i] = fmt.Sprintf("'%s'", sanitized)
}
}
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s IN (%s)", ValidSQL(parmk, "colname"), strings.Join(sanitizedValues, ",")))
} else { } else {
if strings.Contains(val, "match=") { if strings.Contains(val, "match=") {
colval := strings.ReplaceAll(val, "match=", "") colval := strings.ReplaceAll(val, "match=", "")
// Escape single quotes and backslashes for LIKE patterns
// But don't escape wildcards % and _ which are intentional
colval = strings.ReplaceAll(colval, "\\", "\\\\")
colval = strings.ReplaceAll(colval, "'", "''")
if colval != "*" { if colval != "*" {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s ILIKE '%%%s%%'", ValidSQL(parmk, "colname"), ValidSQL(colval, "colvalue"))) sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s ILIKE '%%%s%%'", ValidSQL(parmk, "colname"), colval))
} }
} else if val == "" || val == "0" { } else if val == "" || val == "0" {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("(%[1]s = %[2]s OR %[1]s IS NULL)", ValidSQL(parmk, "colname"), ValidSQL(val, "colvalue"))) // For empty/zero values, treat as literal 0 or empty string with quotes
if val == "0" {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("(%[1]s = 0 OR %[1]s IS NULL)", ValidSQL(parmk, "colname")))
} else {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("(%[1]s = '' OR %[1]s IS NULL)", ValidSQL(parmk, "colname")))
}
} else { } else {
if IsNumeric(val) { if IsNumeric(val) {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s = %s", ValidSQL(parmk, "colname"), ValidSQL(val, "colvalue"))) sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s = %s", ValidSQL(parmk, "colname"), ValidSQL(val, "colvalue")))
@@ -743,16 +776,25 @@ func (h *Handler) mergeHeaderParams(r *http.Request, sqlquery string, variables
kword := fmt.Sprintf("[%s]", k) kword := fmt.Sprintf("[%s]", k)
if strings.Contains(sqlquery, kword) { if strings.Contains(sqlquery, kword) {
sqlquery = strings.ReplaceAll(sqlquery, kword, val) // Sanitize the header value before replacing
sanitized := ValidSQL(val, "colvalue")
sqlquery = strings.ReplaceAll(sqlquery, kword, sanitized)
} }
// Handle special headers // Handle special headers
if strings.Contains(k, "x-fieldfilter-") { if strings.Contains(k, "x-fieldfilter-") {
colname := strings.ReplaceAll(k, "x-fieldfilter-", "") colname := strings.ReplaceAll(k, "x-fieldfilter-", "")
if val == "" || val == "0" { switch val {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("COALESCE(%s, 0) = %s", ValidSQL(colname, "colname"), ValidSQL(val, "colvalue"))) case "0":
} else { sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("COALESCE(%s, 0) = 0", ValidSQL(colname, "colname")))
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s = %s", ValidSQL(colname, "colname"), ValidSQL(val, "colvalue"))) case "":
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("(%[1]s = '' OR %[1]s IS NULL)", ValidSQL(colname, "colname")))
default:
if IsNumeric(val) {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s = %s", ValidSQL(colname, "colname"), ValidSQL(val, "colvalue")))
} else {
sqlquery = sqlQryWhere(sqlquery, fmt.Sprintf("%s = '%s'", ValidSQL(colname, "colname"), ValidSQL(val, "colvalue")))
}
} }
} }
@@ -782,12 +824,15 @@ func (h *Handler) mergeHeaderParams(r *http.Request, sqlquery string, variables
func (h *Handler) replaceMetaVariables(sqlquery string, r *http.Request, userCtx *security.UserContext, metainfo map[string]interface{}, variables map[string]interface{}) string { func (h *Handler) replaceMetaVariables(sqlquery string, r *http.Request, userCtx *security.UserContext, metainfo map[string]interface{}, variables map[string]interface{}) string {
if strings.Contains(sqlquery, "[p_meta_default]") { if strings.Contains(sqlquery, "[p_meta_default]") {
data, _ := json.Marshal(metainfo) data, _ := json.Marshal(metainfo)
sqlquery = strings.ReplaceAll(sqlquery, "[p_meta_default]", fmt.Sprintf("'%s'::jsonb", string(data))) dataStr := strings.ReplaceAll(string(data), "$META$", "/*META*/")
sqlquery = strings.ReplaceAll(sqlquery, "[p_meta_default]", fmt.Sprintf("$META$%s$META$::jsonb", dataStr))
} }
if strings.Contains(sqlquery, "[json_variables]") { if strings.Contains(sqlquery, "[json_variables]") {
data, _ := json.Marshal(variables) data, _ := json.Marshal(variables)
sqlquery = strings.ReplaceAll(sqlquery, "[json_variables]", fmt.Sprintf("'%s'::jsonb", string(data))) dataStr := strings.ReplaceAll(string(data), "$VAR$", "/*VAR*/")
sqlquery = strings.ReplaceAll(sqlquery, "[json_variables]", fmt.Sprintf("$VAR$%s$VAR$::jsonb", dataStr))
} }
if strings.Contains(sqlquery, "[rid_user]") { if strings.Contains(sqlquery, "[rid_user]") {
@@ -795,7 +840,7 @@ func (h *Handler) replaceMetaVariables(sqlquery string, r *http.Request, userCtx
} }
if strings.Contains(sqlquery, "[user]") { if strings.Contains(sqlquery, "[user]") {
sqlquery = strings.ReplaceAll(sqlquery, "[user]", fmt.Sprintf("'%s'", userCtx.UserName)) sqlquery = strings.ReplaceAll(sqlquery, "[user]", fmt.Sprintf("$USR$%s$USR$", strings.ReplaceAll(userCtx.UserName, "$USR$", "/*USR*/")))
} }
if strings.Contains(sqlquery, "[rid_session]") { if strings.Contains(sqlquery, "[rid_session]") {
@@ -806,7 +851,7 @@ func (h *Handler) replaceMetaVariables(sqlquery string, r *http.Request, userCtx
} }
if strings.Contains(sqlquery, "[method]") { if strings.Contains(sqlquery, "[method]") {
sqlquery = strings.ReplaceAll(sqlquery, "[method]", r.Method) sqlquery = strings.ReplaceAll(sqlquery, "[method]", fmt.Sprintf("$M$%s$M$", strings.ReplaceAll(r.Method, "$M$", "/*M*/")))
} }
if strings.Contains(sqlquery, "[post_body]") { if strings.Contains(sqlquery, "[post_body]") {
@@ -819,7 +864,7 @@ func (h *Handler) replaceMetaVariables(sqlquery string, r *http.Request, userCtx
} }
} }
} }
sqlquery = strings.ReplaceAll(sqlquery, "[post_body]", fmt.Sprintf("'%s'", bodystr)) sqlquery = strings.ReplaceAll(sqlquery, "[post_body]", fmt.Sprintf("$PBODY$%s$PBODY$", strings.ReplaceAll(bodystr, "$PBODY$", "/*PBODY*/")))
} }
return sqlquery return sqlquery
@@ -859,19 +904,23 @@ func ValidSQL(input, mode string) string {
reg := regexp.MustCompile(`[^a-zA-Z0-9_\.]`) reg := regexp.MustCompile(`[^a-zA-Z0-9_\.]`)
return reg.ReplaceAllString(input, "") return reg.ReplaceAllString(input, "")
case "colvalue": case "colvalue":
// For column values, escape single quotes // For column values, escape single quotes and backslashes
return strings.ReplaceAll(input, "'", "''") // Note: Backslashes must be escaped first, then single quotes
result := strings.ReplaceAll(input, "\\", "\\\\")
result = strings.ReplaceAll(result, "'", "''")
return result
case "select": case "select":
// For SELECT clauses, be more permissive but still safe // For SELECT clauses, be more permissive but still safe
// Remove semicolons and common SQL injection patterns // Remove semicolons and common SQL injection patterns (case-insensitive)
dangerous := []string{";", "--", "/*", "*/", "xp_", "sp_", "DROP ", "DELETE ", "TRUNCATE ", "UPDATE ", "INSERT "} dangerous := []string{
result := input ";", "--", "/\\*", "\\*/", "xp_", "sp_",
for _, d := range dangerous { "drop ", "delete ", "truncate ", "update ", "insert ",
result = strings.ReplaceAll(result, d, "") "exec ", "execute ", "union ", "declare ", "alter ", "create ",
result = strings.ReplaceAll(result, strings.ToLower(d), "")
result = strings.ReplaceAll(result, strings.ToUpper(d), "")
} }
return result // Build a single regex pattern with all dangerous keywords
pattern := "(?i)(" + strings.Join(dangerous, "|") + ")"
re := regexp.MustCompile(pattern)
return re.ReplaceAllString(input, "")
default: default:
return input return input
} }

View File

@@ -7,8 +7,8 @@ A pluggable metrics collection system with Prometheus implementation.
```go ```go
import "github.com/bitechdev/ResolveSpec/pkg/metrics" import "github.com/bitechdev/ResolveSpec/pkg/metrics"
// Initialize Prometheus provider // Initialize Prometheus provider with default config
provider := metrics.NewPrometheusProvider() provider := metrics.NewPrometheusProvider(nil)
metrics.SetProvider(provider) metrics.SetProvider(provider)
// Apply middleware to your router // Apply middleware to your router
@@ -18,6 +18,41 @@ router.Use(provider.Middleware)
http.Handle("/metrics", provider.Handler()) http.Handle("/metrics", provider.Handler())
``` ```
## Configuration
You can customize the metrics provider using a configuration struct:
```go
import "github.com/bitechdev/ResolveSpec/pkg/metrics"
// Create custom configuration
config := &metrics.Config{
Enabled: true,
Provider: "prometheus",
Namespace: "myapp", // Prefix all metrics with "myapp_"
HTTPRequestBuckets: []float64{0.01, 0.05, 0.1, 0.5, 1, 2, 5},
DBQueryBuckets: []float64{0.001, 0.01, 0.05, 0.1, 0.5, 1},
}
// Initialize with custom config
provider := metrics.NewPrometheusProvider(config)
metrics.SetProvider(provider)
```
### Configuration Options
| Field | Type | Default | Description |
|-------|------|---------|-------------|
| `Enabled` | `bool` | `true` | Enable/disable metrics collection |
| `Provider` | `string` | `"prometheus"` | Metrics provider type |
| `Namespace` | `string` | `""` | Prefix for all metric names |
| `HTTPRequestBuckets` | `[]float64` | See below | Histogram buckets for HTTP duration (seconds) |
| `DBQueryBuckets` | `[]float64` | See below | Histogram buckets for DB query duration (seconds) |
**Default HTTP Request Buckets:** `[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]`
**Default DB Query Buckets:** `[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]`
## Provider Interface ## Provider Interface
The package uses a provider interface, allowing you to plug in different metric systems: The package uses a provider interface, allowing you to plug in different metric systems:
@@ -87,6 +122,13 @@ When using `PrometheusProvider`, the following metrics are available:
| `cache_hits_total` | Counter | provider | Total cache hits | | `cache_hits_total` | Counter | provider | Total cache hits |
| `cache_misses_total` | Counter | provider | Total cache misses | | `cache_misses_total` | Counter | provider | Total cache misses |
| `cache_size_items` | Gauge | provider | Current cache size | | `cache_size_items` | Gauge | provider | Current cache size |
| `events_published_total` | Counter | source, event_type | Total events published |
| `events_processed_total` | Counter | source, event_type, status | Total events processed |
| `event_processing_duration_seconds` | Histogram | source, event_type | Event processing duration |
| `event_queue_size` | Gauge | - | Current event queue size |
| `panics_total` | Counter | method | Total panics recovered |
**Note:** If a custom `Namespace` is configured, all metric names will be prefixed with `{namespace}_`.
## Prometheus Queries ## Prometheus Queries
@@ -148,6 +190,8 @@ metrics.SetProvider(&CustomProvider{})
## Complete Example ## Complete Example
### Basic Usage
```go ```go
package main package main
@@ -162,8 +206,8 @@ import (
) )
func main() { func main() {
// Initialize metrics // Initialize metrics with default config
provider := metrics.NewPrometheusProvider() provider := metrics.NewPrometheusProvider(nil)
metrics.SetProvider(provider) metrics.SetProvider(provider)
// Create router // Create router
@@ -198,6 +242,42 @@ func getUsersHandler(w http.ResponseWriter, r *http.Request) {
} }
``` ```
### With Custom Configuration
```go
package main
import (
"log"
"net/http"
"github.com/bitechdev/ResolveSpec/pkg/metrics"
"github.com/gorilla/mux"
)
func main() {
// Custom metrics configuration
metricsConfig := &metrics.Config{
Enabled: true,
Provider: "prometheus",
Namespace: "myapp",
// Custom buckets optimized for your application
HTTPRequestBuckets: []float64{0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10},
DBQueryBuckets: []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1},
}
// Initialize with custom config
provider := metrics.NewPrometheusProvider(metricsConfig)
metrics.SetProvider(provider)
router := mux.NewRouter()
router.Use(provider.Middleware)
router.Handle("/metrics", provider.Handler())
log.Fatal(http.ListenAndServe(":8080", router))
}
```
## Docker Compose Example ## Docker Compose Example
```yaml ```yaml

46
pkg/metrics/config.go Normal file
View File

@@ -0,0 +1,46 @@
package metrics
// Config holds configuration for the metrics provider
type Config struct {
// Enabled determines whether metrics collection is enabled
Enabled bool `mapstructure:"enabled"`
// Provider specifies which metrics provider to use (prometheus, noop)
Provider string `mapstructure:"provider"`
// Namespace is an optional prefix for all metric names
Namespace string `mapstructure:"namespace"`
// HTTPRequestBuckets defines histogram buckets for HTTP request duration (in seconds)
// Default: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
HTTPRequestBuckets []float64 `mapstructure:"http_request_buckets"`
// DBQueryBuckets defines histogram buckets for database query duration (in seconds)
// Default: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]
DBQueryBuckets []float64 `mapstructure:"db_query_buckets"`
}
// DefaultConfig returns a Config with sensible defaults
func DefaultConfig() *Config {
return &Config{
Enabled: true,
Provider: "prometheus",
// HTTP requests typically take longer than DB queries
HTTPRequestBuckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
// DB queries are usually faster
DBQueryBuckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
}
}
// ApplyDefaults fills in any missing values with defaults
func (c *Config) ApplyDefaults() {
if c.Provider == "" {
c.Provider = "prometheus"
}
if len(c.HTTPRequestBuckets) == 0 {
c.HTTPRequestBuckets = []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}
}
if len(c.DBQueryBuckets) == 0 {
c.DBQueryBuckets = []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5}
}
}

View File

@@ -0,0 +1,64 @@
package metrics_test
import (
"fmt"
"github.com/bitechdev/ResolveSpec/pkg/metrics"
)
// ExampleNewPrometheusProvider_default demonstrates using default configuration
func ExampleNewPrometheusProvider_default() {
// Initialize with default configuration
provider := metrics.NewPrometheusProvider(nil)
metrics.SetProvider(provider)
fmt.Println("Provider initialized with defaults")
// Output: Provider initialized with defaults
}
// ExampleNewPrometheusProvider_custom demonstrates using custom configuration
func ExampleNewPrometheusProvider_custom() {
// Create custom configuration
config := &metrics.Config{
Enabled: true,
Provider: "prometheus",
Namespace: "myapp",
HTTPRequestBuckets: []float64{0.01, 0.05, 0.1, 0.5, 1, 2, 5},
DBQueryBuckets: []float64{0.001, 0.01, 0.05, 0.1, 0.5, 1},
}
// Initialize with custom configuration
provider := metrics.NewPrometheusProvider(config)
metrics.SetProvider(provider)
fmt.Println("Provider initialized with custom config")
// Output: Provider initialized with custom config
}
// ExampleDefaultConfig demonstrates getting default configuration
func ExampleDefaultConfig() {
config := metrics.DefaultConfig()
fmt.Printf("Default provider: %s\n", config.Provider)
fmt.Printf("Default enabled: %v\n", config.Enabled)
// Output:
// Default provider: prometheus
// Default enabled: true
}
// ExampleConfig_ApplyDefaults demonstrates applying defaults to partial config
func ExampleConfig_ApplyDefaults() {
// Create partial configuration
config := &metrics.Config{
Namespace: "myapp",
// Other fields will be filled with defaults
}
// Apply defaults
config.ApplyDefaults()
fmt.Printf("Provider: %s\n", config.Provider)
fmt.Printf("Namespace: %s\n", config.Namespace)
// Output:
// Provider: prometheus
// Namespace: myapp
}

View File

@@ -20,23 +20,44 @@ type PrometheusProvider struct {
cacheHits *prometheus.CounterVec cacheHits *prometheus.CounterVec
cacheMisses *prometheus.CounterVec cacheMisses *prometheus.CounterVec
cacheSize *prometheus.GaugeVec cacheSize *prometheus.GaugeVec
eventPublished *prometheus.CounterVec
eventProcessed *prometheus.CounterVec
eventDuration *prometheus.HistogramVec
eventQueueSize prometheus.Gauge
panicsTotal *prometheus.CounterVec panicsTotal *prometheus.CounterVec
} }
// NewPrometheusProvider creates a new Prometheus metrics provider // NewPrometheusProvider creates a new Prometheus metrics provider
func NewPrometheusProvider() *PrometheusProvider { // If cfg is nil, default configuration will be used
func NewPrometheusProvider(cfg *Config) *PrometheusProvider {
// Use default config if none provided
if cfg == nil {
cfg = DefaultConfig()
} else {
// Apply defaults for any missing values
cfg.ApplyDefaults()
}
// Helper to add namespace prefix if configured
metricName := func(name string) string {
if cfg.Namespace != "" {
return cfg.Namespace + "_" + name
}
return name
}
return &PrometheusProvider{ return &PrometheusProvider{
requestDuration: promauto.NewHistogramVec( requestDuration: promauto.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Name: "http_request_duration_seconds", Name: metricName("http_request_duration_seconds"),
Help: "HTTP request duration in seconds", Help: "HTTP request duration in seconds",
Buckets: prometheus.DefBuckets, Buckets: cfg.HTTPRequestBuckets,
}, },
[]string{"method", "path", "status"}, []string{"method", "path", "status"},
), ),
requestTotal: promauto.NewCounterVec( requestTotal: promauto.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Name: "http_requests_total", Name: metricName("http_requests_total"),
Help: "Total number of HTTP requests", Help: "Total number of HTTP requests",
}, },
[]string{"method", "path", "status"}, []string{"method", "path", "status"},
@@ -44,49 +65,77 @@ func NewPrometheusProvider() *PrometheusProvider {
requestsInFlight: promauto.NewGauge( requestsInFlight: promauto.NewGauge(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Name: "http_requests_in_flight", Name: metricName("http_requests_in_flight"),
Help: "Current number of HTTP requests being processed", Help: "Current number of HTTP requests being processed",
}, },
), ),
dbQueryDuration: promauto.NewHistogramVec( dbQueryDuration: promauto.NewHistogramVec(
prometheus.HistogramOpts{ prometheus.HistogramOpts{
Name: "db_query_duration_seconds", Name: metricName("db_query_duration_seconds"),
Help: "Database query duration in seconds", Help: "Database query duration in seconds",
Buckets: prometheus.DefBuckets, Buckets: cfg.DBQueryBuckets,
}, },
[]string{"operation", "table"}, []string{"operation", "table"},
), ),
dbQueryTotal: promauto.NewCounterVec( dbQueryTotal: promauto.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Name: "db_queries_total", Name: metricName("db_queries_total"),
Help: "Total number of database queries", Help: "Total number of database queries",
}, },
[]string{"operation", "table", "status"}, []string{"operation", "table", "status"},
), ),
cacheHits: promauto.NewCounterVec( cacheHits: promauto.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Name: "cache_hits_total", Name: metricName("cache_hits_total"),
Help: "Total number of cache hits", Help: "Total number of cache hits",
}, },
[]string{"provider"}, []string{"provider"},
), ),
cacheMisses: promauto.NewCounterVec( cacheMisses: promauto.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Name: "cache_misses_total", Name: metricName("cache_misses_total"),
Help: "Total number of cache misses", Help: "Total number of cache misses",
}, },
[]string{"provider"}, []string{"provider"},
), ),
cacheSize: promauto.NewGaugeVec( cacheSize: promauto.NewGaugeVec(
prometheus.GaugeOpts{ prometheus.GaugeOpts{
Name: "cache_size_items", Name: metricName("cache_size_items"),
Help: "Number of items in cache", Help: "Number of items in cache",
}, },
[]string{"provider"}, []string{"provider"},
), ),
eventPublished: promauto.NewCounterVec(
prometheus.CounterOpts{
Name: metricName("events_published_total"),
Help: "Total number of events published",
},
[]string{"source", "event_type"},
),
eventProcessed: promauto.NewCounterVec(
prometheus.CounterOpts{
Name: metricName("events_processed_total"),
Help: "Total number of events processed",
},
[]string{"source", "event_type", "status"},
),
eventDuration: promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: metricName("event_processing_duration_seconds"),
Help: "Event processing duration in seconds",
Buckets: cfg.DBQueryBuckets, // Events are typically fast like DB queries
},
[]string{"source", "event_type"},
),
eventQueueSize: promauto.NewGauge(
prometheus.GaugeOpts{
Name: metricName("event_queue_size"),
Help: "Current number of events in queue",
},
),
panicsTotal: promauto.NewCounterVec( panicsTotal: promauto.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
Name: "panics_total", Name: metricName("panics_total"),
Help: "Total number of panics", Help: "Total number of panics",
}, },
[]string{"method"}, []string{"method"},
@@ -153,6 +202,22 @@ func (p *PrometheusProvider) UpdateCacheSize(provider string, size int64) {
p.cacheSize.WithLabelValues(provider).Set(float64(size)) p.cacheSize.WithLabelValues(provider).Set(float64(size))
} }
// RecordEventPublished implements Provider interface
func (p *PrometheusProvider) RecordEventPublished(source, eventType string) {
p.eventPublished.WithLabelValues(source, eventType).Inc()
}
// RecordEventProcessed implements Provider interface
func (p *PrometheusProvider) RecordEventProcessed(source, eventType, status string, duration time.Duration) {
p.eventProcessed.WithLabelValues(source, eventType, status).Inc()
p.eventDuration.WithLabelValues(source, eventType).Observe(duration.Seconds())
}
// UpdateEventQueueSize implements Provider interface
func (p *PrometheusProvider) UpdateEventQueueSize(size int64) {
p.eventQueueSize.Set(float64(size))
}
// RecordPanic implements the Provider interface // RecordPanic implements the Provider interface
func (p *PrometheusProvider) RecordPanic(methodName string) { func (p *PrometheusProvider) RecordPanic(methodName string) {
p.panicsTotal.WithLabelValues(methodName).Inc() p.panicsTotal.WithLabelValues(methodName).Inc()