ResolveSpec/pkg/middleware/README.md

Middleware Package

HTTP middleware utilities for security and performance.

Table of Contents

1. Rate Limiting
2. Client Request Queue
3. Request Size Limits
4. Input Sanitization

---

Rate Limiting

Production-grade rate limiting using token bucket algorithm.

Quick Start

import "github.com/bitechdev/ResolveSpec/pkg/middleware"

// Create rate limiter: 100 requests per second, burst of 20
rateLimiter := middleware.NewRateLimiter(100, 20)

// Apply to all routes
router.Use(rateLimiter.Middleware)

Basic Usage

package main

import (
    "log"
    "net/http"

    "github.com/bitechdev/ResolveSpec/pkg/middleware"
    "github.com/gorilla/mux"
)

func main() {
    router := mux.NewRouter()

    // Rate limit: 10 requests per second, burst of 5
    rateLimiter := middleware.NewRateLimiter(10, 5)
    router.Use(rateLimiter.Middleware)

    router.HandleFunc("/api/data", dataHandler)

    log.Fatal(http.ListenAndServe(":8080", router))
}

Custom Key Extraction

By default, rate limiting is per IP address. Customize the key:

// Rate limit by User ID from header
keyFunc := func(r *http.Request) string {
    userID := r.Header.Get("X-User-ID")
    if userID == "" {
        return r.RemoteAddr // Fallback to IP
    }
    return "user:" + userID
}

router.Use(rateLimiter.MiddlewareWithKeyFunc(keyFunc))

Advanced Key Functions

By API Key:

keyFunc := func(r *http.Request) string {
    apiKey := r.Header.Get("X-API-Key")
    if apiKey == "" {
        return r.RemoteAddr
    }
    return "api:" + apiKey
}

By Authenticated User:

keyFunc := func(r *http.Request) string {
    // Extract from JWT or session
    user := getUserFromContext(r.Context())
    if user != nil {
        return "user:" + user.ID
    }
    return r.RemoteAddr
}

By Path + User:

keyFunc := func(r *http.Request) string {
    user := getUserFromContext(r.Context())
    if user != nil {
        return fmt.Sprintf("user:%s:path:%s", user.ID, r.URL.Path)
    }
    return r.URL.Path + ":" + r.RemoteAddr
}

Different Limits Per Route

func main() {
    router := mux.NewRouter()

    // Public endpoints: 10 rps
    publicLimiter := middleware.NewRateLimiter(10, 5)

    // API endpoints: 100 rps
    apiLimiter := middleware.NewRateLimiter(100, 20)

    // Admin endpoints: 1000 rps
    adminLimiter := middleware.NewRateLimiter(1000, 50)

    // Apply different limiters to subrouters
    publicRouter := router.PathPrefix("/public").Subrouter()
    publicRouter.Use(publicLimiter.Middleware)

    apiRouter := router.PathPrefix("/api").Subrouter()
    apiRouter.Use(apiLimiter.Middleware)

    adminRouter := router.PathPrefix("/admin").Subrouter()
    adminRouter.Use(adminLimiter.Middleware)
}

Rate Limit Response

When rate limited, clients receive:

HTTP/1.1 429 Too Many Requests
Content-Type: text/plain

{"error":"rate_limit_exceeded","message":"Too many requests"}

Configuration Examples

Tight Rate Limit (Anti-abuse):

// 1 request per second, burst of 3
rateLimiter := middleware.NewRateLimiter(1, 3)

Moderate Rate Limit (Standard API):

// 100 requests per second, burst of 20
rateLimiter := middleware.NewRateLimiter(100, 20)

Generous Rate Limit (Internal Services):

// 1000 requests per second, burst of 100
rateLimiter := middleware.NewRateLimiter(1000, 100)

Time-based Limits:

// 60 requests per minute = 1 request per second
rateLimiter := middleware.NewRateLimiter(1, 10)

// 1000 requests per hour ≈ 0.28 requests per second
rateLimiter := middleware.NewRateLimiter(0.28, 50)

Understanding Burst

The burst parameter allows short bursts above the rate:

// Rate: 10 rps, Burst: 5
// Allows up to 5 requests immediately, then 10/second
rateLimiter := middleware.NewRateLimiter(10, 5)

Bucket fills at rate: 10 tokens/second Bucket capacity: 5 tokens Request consumes: 1 token

Example traffic pattern:

• T=0s: 5 requests → ✅ All allowed (burst)
• T=0.1s: 1 request → ❌ Denied (bucket empty)
• T=0.5s: 1 request → ✅ Allowed (bucket refilled 0.5 tokens)
• T=1s: 1 request → ✅ Allowed (bucket has ~1 token)

Cleanup Behavior

The rate limiter automatically cleans up inactive limiters every 5 minutes to prevent memory leaks.

Performance Characteristics

• Memory: ~100 bytes per active limiter
• Throughput: >1M requests/second
• Latency: <1μs per request
• Concurrency: Lock-free for rate checks

Production Deployment

With Reverse Proxy:

// Use X-Forwarded-For or X-Real-IP
keyFunc := func(r *http.Request) string {
    // Check proxy headers first
    if ip := r.Header.Get("X-Forwarded-For"); ip != "" {
        return strings.Split(ip, ",")[0]
    }
    if ip := r.Header.Get("X-Real-IP"); ip != "" {
        return ip
    }
    return r.RemoteAddr
}

router.Use(rateLimiter.MiddlewareWithKeyFunc(keyFunc))

Environment-based Configuration:

import "os"

func getRateLimiter() *middleware.RateLimiter {
    rps := getEnvFloat("RATE_LIMIT_RPS", 100)
    burst := getEnvInt("RATE_LIMIT_BURST", 20)
    return middleware.NewRateLimiter(rps, burst)
}

Testing Rate Limits

# Send 10 requests rapidly
for i in {1..10}; do
  curl -w "Status: %{http_code}\n" http://localhost:8080/api/data
done

Expected output:

Status: 200  # Request 1-5 (within burst)
Status: 200
Status: 200
Status: 200
Status: 200
Status: 429  # Request 6-10 (rate limited)
Status: 429
Status: 429
Status: 429
Status: 429

Complete Example

package main

import (
    "encoding/json"
    "log"
    "net/http"
    "os"
    "strconv"

    "github.com/bitechdev/ResolveSpec/pkg/middleware"
    "github.com/gorilla/mux"
)

func main() {
    // Configuration from environment
    rps, _ := strconv.ParseFloat(os.Getenv("RATE_LIMIT_RPS"), 64)
    if rps == 0 {
        rps = 100 // Default
    }

    burst, _ := strconv.Atoi(os.Getenv("RATE_LIMIT_BURST"))
    if burst == 0 {
        burst = 20 // Default
    }

    // Create rate limiter
    rateLimiter := middleware.NewRateLimiter(rps, burst)

    // Custom key extraction
    keyFunc := func(r *http.Request) string {
        // Try API key first
        if apiKey := r.Header.Get("X-API-Key"); apiKey != "" {
            return "api:" + apiKey
        }
        // Try authenticated user
        if userID := r.Header.Get("X-User-ID"); userID != "" {
            return "user:" + userID
        }
        // Fall back to IP
        if ip := r.Header.Get("X-Forwarded-For"); ip != "" {
            return ip
        }
        return r.RemoteAddr
    }

    // Create router
    router := mux.NewRouter()

    // Apply rate limiting
    router.Use(rateLimiter.MiddlewareWithKeyFunc(keyFunc))

    // Routes
    router.HandleFunc("/api/data", dataHandler)
    router.HandleFunc("/health", healthHandler)

    log.Printf("Starting server with rate limit: %.1f rps, burst: %d", rps, burst)
    log.Fatal(http.ListenAndServe(":8080", router))
}

func dataHandler(w http.ResponseWriter, r *http.Request) {
    json.NewEncoder(w).Encode(map[string]string{
        "message": "Data endpoint",
    })
}

func healthHandler(w http.ResponseWriter, r *http.Request) {
    w.WriteHeader(http.StatusOK)
    w.Write([]byte("OK"))
}

Best Practices

1. Set Appropriate Limits: Consider your backend capacity

   • Database: Can it handle X queries/second?
   • External APIs: What are their rate limits?
   • Server resources: CPU, memory, connections

2. Use Burst Wisely: Allow legitimate traffic spikes

   • Too low: Reject valid bursts
   • Too high: Allow abuse

3. Monitor Rate Limits: Track how often limits are hit

   // Log rate limit events
   if rateLimited {
       log.Printf("Rate limited: %s", clientKey)
   }
   

4. Provide Feedback: Include rate limit headers (future enhancement)

   X-RateLimit-Limit: 100
   X-RateLimit-Remaining: 95
   X-RateLimit-Reset: 1640000000
   

5. Tiered Limits: Different limits for different user tiers

   func getRateLimiter(userTier string) *middleware.RateLimiter {
       switch userTier {
       case "premium":
           return middleware.NewRateLimiter(1000, 100)
       case "standard":
           return middleware.NewRateLimiter(100, 20)
       default:
           return middleware.NewRateLimiter(10, 5)
       }
   }
   

---

Client Request Queue

ClientQueue smooths bursts (for example a page load that fires ~15 requests at once) by limiting how many requests each client runs concurrently and queueing the rest first-in-first-out. Unlike the rate limiter it never rejects a request that can be served shortly.

q := middleware.NewClientQueue(middleware.ClientQueueConfig{
    MaxConcurrent: 4,                // running at once, per client
    MaxQueue:      50,               // waiting, per client; beyond this -> 429
    MaxWait:       30 * time.Second, // waiting too long -> 503 + Retry-After
})
defer q.Close()

router.Use(q.Middleware) // gorilla/mux; or wrap any http.Handler

Adding it to restheadspec / resolvespec

Both packages' SetupMuxRoutes and SetupBunRouterRoutes take one middleware and apply it to every route, so pass the queue there. Use Chain to combine it with auth (first is outermost, so requests are authenticated before they can take a queue slot):

q := middleware.NewClientQueue(middleware.ClientQueueConfig{MaxConcurrent: 4})
defer q.Close()

mw := middleware.Chain(authMiddleware, q.Middleware) // authMiddleware may be nil

restheadspec.SetupMuxRoutes(muxRouter, headHandler, mw)
resolvespec.SetupMuxRoutes(muxRouter, resolveHandler, mw)
// or SetupBunRouterRoutes(router, handler, mw)

Share one ClientQueue across packages to give a client a single limit over all of them.

Client identification

The first of these that is present is used, in order:

1. X-Client-Id header
2. Authorization header
3. the built-in server session (security.GetSessionID from the request context, else the session cookie)
4. the client IP

Nothing is required from the client: without an id it falls back to the session, then to the IP. Secrets are hashed and never stored. Ids longer than 128 characters are ignored.

The session is only in the request context if the security auth middleware has run first, so put auth before the queue with Chain (below). Without it the cookie is still used.

To get one queue per browser tab (rather than per session), have the client send an id generated once per tab; the server cannot read sessionStorage itself:

const id = sessionStorage.clientId ??= crypto.randomUUID();
fetch(url, { headers: { "X-Client-Id": id } });

Metrics

Registered on the default Prometheus registry (the same one pkg/metrics and dbmanager use), with no per-client labels so cardinality stays bounded.

Metric	Type	Meaning
clientqueue_requests_total{result}	counter	immediate, queued, rejected_full, timeout, canceled
clientqueue_wait_seconds	histogram	wait for a slot; 0 for requests that ran immediately
clientqueue_wait_max_seconds	gauge	longest wait since process start
clientqueue_burst_size	histogram	peak outstanding requests per client busy period
clientqueue_burst_max	gauge	largest burst since process start
clientqueue_active / clientqueue_queue_depth	gauge	running / waiting now
clientqueue_clients	gauge	clients currently tracked

A burst is one client's busy period: the peak number of its requests running plus waiting between going from idle to busy and back to idle. A page load firing 15 requests at once is a burst of 15.

# average burst size
rate(clientqueue_burst_size_sum[5m]) / rate(clientqueue_burst_size_count[5m])
# bursts larger than the concurrency limit (need queueing)
1 - (sum(rate(clientqueue_burst_size_bucket{le="10"}[5m])) / sum(rate(clientqueue_burst_size_count[5m])))
# average and p95 wait
rate(clientqueue_wait_seconds_sum[5m]) / rate(clientqueue_wait_seconds_count[5m])
histogram_quantile(0.95, sum(rate(clientqueue_wait_seconds_bucket[5m])) by (le))
# share of requests that had to queue
sum(rate(clientqueue_requests_total{result="queued"}[5m])) / sum(rate(clientqueue_requests_total[5m]))

Use these to pick MaxConcurrent: if the p95 burst is well above it and waits are short, it is doing its job; if waits grow, the limit is too low or the database is the bottleneck. The _max gauges reset on restart; use the histograms for anything over time. A client that never goes idle produces one long busy period, so its burst is only recorded when it finally drains.

Behaviour

• Queued requests are dropped if the client disconnects, so abandoned requests never run.
• CORS preflights (OPTIONS) and connection upgrades (websockets) bypass the queue.
• Idle clients are forgotten after IdleTimeout (default 5m).
• The client id is client-supplied, so a hostile client can rotate ids to get more slots. It smooths well-behaved clients; it is not an abuse control. Combine it with RateLimiter for that.
• Queue time counts against your own request timeouts; keep MaxWait below them.

Request Size Limits

Protect against oversized request bodies with configurable size limits.

Quick Start

import "github.com/bitechdev/ResolveSpec/pkg/middleware"

// Default: 10MB limit
sizeLimiter := middleware.NewRequestSizeLimiter(0)
router.Use(sizeLimiter.Middleware)

Custom Size Limit

// 5MB limit
sizeLimiter := middleware.NewRequestSizeLimiter(5 * 1024 * 1024)
router.Use(sizeLimiter.Middleware)

// Or use constants
sizeLimiter := middleware.NewRequestSizeLimiter(middleware.Size5MB)

Available Size Constants

middleware.Size1MB    // 1 MB
middleware.Size5MB    // 5 MB
middleware.Size10MB   // 10 MB (default)
middleware.Size50MB   // 50 MB
middleware.Size100MB  // 100 MB

Different Limits Per Route

func main() {
    router := mux.NewRouter()

    // File upload endpoint: 50MB
    uploadLimiter := middleware.NewRequestSizeLimiter(middleware.Size50MB)
    uploadRouter := router.PathPrefix("/upload").Subrouter()
    uploadRouter.Use(uploadLimiter.Middleware)

    // API endpoints: 1MB
    apiLimiter := middleware.NewRequestSizeLimiter(middleware.Size1MB)
    apiRouter := router.PathPrefix("/api").Subrouter()
    apiRouter.Use(apiLimiter.Middleware)
}

Dynamic Size Limits

// Custom size based on request
sizeFunc := func(r *http.Request) int64 {
    // Premium users get 50MB
    if isPremiumUser(r) {
        return middleware.Size50MB
    }
    // Free users get 5MB
    return middleware.Size5MB
}

router.Use(sizeLimiter.MiddlewareWithCustomSize(sizeFunc))

By Content-Type:

sizeFunc := func(r *http.Request) int64 {
    contentType := r.Header.Get("Content-Type")

    switch {
    case strings.Contains(contentType, "multipart/form-data"):
        return middleware.Size50MB // File uploads
    case strings.Contains(contentType, "application/json"):
        return middleware.Size1MB // JSON APIs
    default:
        return middleware.Size10MB // Default
    }
}

Error Response

When size limit exceeded:

HTTP/1.1 413 Request Entity Too Large
X-Max-Request-Size: 10485760

http: request body too large

Complete Example

package main

import (
    "log"
    "net/http"

    "github.com/bitechdev/ResolveSpec/pkg/middleware"
    "github.com/gorilla/mux"
)

func main() {
    router := mux.NewRouter()

    // API routes: 1MB limit
    api := router.PathPrefix("/api").Subrouter()
    apiLimiter := middleware.NewRequestSizeLimiter(middleware.Size1MB)
    api.Use(apiLimiter.Middleware)
    api.HandleFunc("/users", createUserHandler).Methods("POST")

    // Upload routes: 50MB limit
    upload := router.PathPrefix("/upload").Subrouter()
    uploadLimiter := middleware.NewRequestSizeLimiter(middleware.Size50MB)
    upload.Use(uploadLimiter.Middleware)
    upload.HandleFunc("/file", uploadFileHandler).Methods("POST")

    log.Fatal(http.ListenAndServe(":8080", router))
}

---

Input Sanitization

Protect against XSS, injection attacks, and malicious input.

Quick Start

import "github.com/bitechdev/ResolveSpec/pkg/middleware"

// Default sanitizer (safe defaults)
sanitizer := middleware.DefaultSanitizer()
router.Use(sanitizer.Middleware)

Sanitizer Types

Default Sanitizer (Recommended):

sanitizer := middleware.DefaultSanitizer()
// ✓ Escapes HTML entities
// ✓ Removes null bytes
// ✓ Removes control characters
// ✓ Blocks XSS patterns (script tags, event handlers)
// ✗ Does not strip HTML (allows legitimate content)

Strict Sanitizer:

sanitizer := middleware.StrictSanitizer()
// ✓ All default features
// ✓ Strips ALL HTML tags
// ✓ Max string length: 10,000 chars

Custom Configuration

sanitizer := &middleware.Sanitizer{
    StripHTML:          true,  // Remove HTML tags
    EscapeHTML:         false, // Don't escape (already stripped)
    RemoveNullBytes:    true,  // Remove \x00
    RemoveControlChars: true,  // Remove dangerous control chars
    MaxStringLength:    5000,  // Limit to 5000 chars

    // Block patterns (regex)
    BlockPatterns: []*regexp.Regexp{
        regexp.MustCompile(`(?i)<script`),
        regexp.MustCompile(`(?i)javascript:`),
    },

    // Custom sanitization function
    CustomSanitizer: func(s string) string {
        // Your custom logic
        return strings.ToLower(s)
    },
}

router.Use(sanitizer.Middleware)

What Gets Sanitized

Automatic (via middleware):

• Query parameters
• Headers (User-Agent, Referer, X-Forwarded-For, X-Real-IP)

Manual (in your handler):

• Request body (JSON, form data)
• Database queries
• File names

Manual Sanitization

String Values:

sanitizer := middleware.DefaultSanitizer()

// Sanitize user input
username := sanitizer.Sanitize(r.FormValue("username"))
email := sanitizer.Sanitize(r.FormValue("email"))

Map/JSON Data:

var data map[string]interface{}
json.Unmarshal(body, &data)

// Sanitize all string values recursively
sanitizedData := sanitizer.SanitizeMap(data)

Nested Structures:

type User struct {
    Name    string
    Email   string
    Bio     string
    Profile map[string]interface{}
}

// After unmarshaling
user.Name = sanitizer.Sanitize(user.Name)
user.Email = sanitizer.Sanitize(user.Email)
user.Bio = sanitizer.Sanitize(user.Bio)
user.Profile = sanitizer.SanitizeMap(user.Profile)

Specialized Sanitizers

Filenames:

import "github.com/bitechdev/ResolveSpec/pkg/middleware"

filename := middleware.SanitizeFilename(uploadedFilename)
// Removes: .., /, \, null bytes
// Limits: 255 characters

Emails:

email := middleware.SanitizeEmail(" USER@EXAMPLE.COM ")
// Result: "user@example.com"
// Trims, lowercases, removes null bytes

URLs:

url := middleware.SanitizeURL(userInput)
// Blocks: javascript:, data: protocols
// Removes: null bytes

Blocked Patterns (Default)

The default sanitizer blocks:

1. Script tags: <script>...</script>
2. JavaScript protocol: javascript:alert(1)
3. Event handlers: onclick="...", onerror="..."
4. Iframes: <iframe src="...">
5. Objects: <object data="...">
6. Embeds: <embed src="...">

Security Best Practices

1. Layer Defense:

// Layer 1: Middleware (query params, headers)
router.Use(sanitizer.Middleware)

// Layer 2: Input validation (in handler)
func createUserHandler(w http.ResponseWriter, r *http.Request) {
    var user User
    json.NewDecoder(r.Body).Decode(&user)

    // Sanitize
    user.Name = sanitizer.Sanitize(user.Name)
    user.Email = middleware.SanitizeEmail(user.Email)

    // Validate
    if !isValidEmail(user.Email) {
        http.Error(w, "Invalid email", 400)
        return
    }

    // Use parameterized queries (prevents SQL injection)
    db.Exec("INSERT INTO users (name, email) VALUES (?, ?)",
        user.Name, user.Email)
}

2. Context-Aware Sanitization:

// HTML content (user posts, comments)
sanitizer := middleware.StrictSanitizer()
post.Content = sanitizer.Sanitize(post.Content)

// Structured data (JSON API)
sanitizer := middleware.DefaultSanitizer()
data = sanitizer.SanitizeMap(jsonData)

// Search queries (preserve special chars)
query = middleware.SanitizeFilename(searchTerm) // Light sanitization

3. Output Encoding:

// When rendering HTML
import "html/template"

tmpl := template.Must(template.New("page").Parse(`
    <h1>{{.Title}}</h1>
    <p>{{.Content}}</p>
`))

// template.HTML automatically escapes
tmpl.Execute(w, data)

Complete Example

package main

import (
    "encoding/json"
    "log"
    "net/http"

    "github.com/bitechdev/ResolveSpec/pkg/middleware"
    "github.com/gorilla/mux"
)

func main() {
    router := mux.NewRouter()

    // Apply sanitization middleware
    sanitizer := middleware.DefaultSanitizer()
    router.Use(sanitizer.Middleware)

    router.HandleFunc("/api/users", createUserHandler).Methods("POST")

    log.Fatal(http.ListenAndServe(":8080", router))
}

func createUserHandler(w http.ResponseWriter, r *http.Request) {
    sanitizer := middleware.DefaultSanitizer()

    var user struct {
        Name  string `json:"name"`
        Email string `json:"email"`
        Bio   string `json:"bio"`
    }

    if err := json.NewDecoder(r.Body).Decode(&user); err != nil {
        http.Error(w, "Invalid JSON", 400)
        return
    }

    // Sanitize inputs
    user.Name = sanitizer.Sanitize(user.Name)
    user.Email = middleware.SanitizeEmail(user.Email)
    user.Bio = sanitizer.Sanitize(user.Bio)

    // Validate
    if len(user.Name) == 0 || len(user.Email) == 0 {
        http.Error(w, "Name and email required", 400)
        return
    }

    // Save to database (use parameterized queries!)
    // db.Exec("INSERT INTO users (name, email, bio) VALUES (?, ?, ?)",
    //     user.Name, user.Email, user.Bio)

    w.WriteHeader(http.StatusCreated)
    json.NewEncoder(w).Encode(map[string]string{
        "status": "created",
    })
}

Testing Sanitization

# Test XSS prevention
curl -X POST http://localhost:8080/api/users \
  -H "Content-Type: application/json" \
  -d '{
    "name": "<script>alert(1)</script>John",
    "email": "test@example.com",
    "bio": "My bio with <iframe src=\"evil.com\"></iframe>"
  }'

# Script tags and iframes should be removed

Performance

• Overhead: <1ms per request for typical payloads
• Regex compilation: Done once at initialization
• Safe for production: Minimal performance impact