wdevs/ResolveSpec

Fork 0

mirror of https://github.com/bitechdev/ResolveSpec.git synced 2025-12-13 17:10:36 +00:00

Hein 2a84652dba Middleware enhancements

2025-12-08 08:47:13 +02:00

18 KiB

Raw Blame History

Middleware Package

HTTP middleware utilities for security and performance.

Rate Limiting
Request Size Limits
Input Sanitization

Rate Limiting

Production-grade rate limiting using token bucket algorithm.

Quick Start

import "github.com/bitechdev/ResolveSpec/pkg/middleware"

// Create rate limiter: 100 requests per second, burst of 20
rateLimiter := middleware.NewRateLimiter(100, 20)

// Apply to all routes
router.Use(rateLimiter.Middleware)

Basic Usage

package main

import (
    "log"
    "net/http"

    "github.com/bitechdev/ResolveSpec/pkg/middleware"
    "github.com/gorilla/mux"
)

func main() {
    router := mux.NewRouter()

    // Rate limit: 10 requests per second, burst of 5
    rateLimiter := middleware.NewRateLimiter(10, 5)
    router.Use(rateLimiter.Middleware)

    router.HandleFunc("/api/data", dataHandler)

    log.Fatal(http.ListenAndServe(":8080", router))
}

Custom Key Extraction

By default, rate limiting is per IP address. Customize the key:

// Rate limit by User ID from header
keyFunc := func(r *http.Request) string {
    userID := r.Header.Get("X-User-ID")
    if userID == "" {
        return r.RemoteAddr // Fallback to IP
    }
    return "user:" + userID
}

router.Use(rateLimiter.MiddlewareWithKeyFunc(keyFunc))

Advanced Key Functions

By API Key:

keyFunc := func(r *http.Request) string {
    apiKey := r.Header.Get("X-API-Key")
    if apiKey == "" {
        return r.RemoteAddr
    }
    return "api:" + apiKey
}

By Authenticated User:

keyFunc := func(r *http.Request) string {
    // Extract from JWT or session
    user := getUserFromContext(r.Context())
    if user != nil {
        return "user:" + user.ID
    }
    return r.RemoteAddr
}

By Path + User:

keyFunc := func(r *http.Request) string {
    user := getUserFromContext(r.Context())
    if user != nil {
        return fmt.Sprintf("user:%s:path:%s", user.ID, r.URL.Path)
    }
    return r.URL.Path + ":" + r.RemoteAddr
}

Different Limits Per Route

func main() {
    router := mux.NewRouter()

    // Public endpoints: 10 rps
    publicLimiter := middleware.NewRateLimiter(10, 5)

    // API endpoints: 100 rps
    apiLimiter := middleware.NewRateLimiter(100, 20)

    // Admin endpoints: 1000 rps
    adminLimiter := middleware.NewRateLimiter(1000, 50)

    // Apply different limiters to subrouters
    publicRouter := router.PathPrefix("/public").Subrouter()
    publicRouter.Use(publicLimiter.Middleware)

    apiRouter := router.PathPrefix("/api").Subrouter()
    apiRouter.Use(apiLimiter.Middleware)

    adminRouter := router.PathPrefix("/admin").Subrouter()
    adminRouter.Use(adminLimiter.Middleware)
}

Rate Limit Response

When rate limited, clients receive:

HTTP/1.1 429 Too Many Requests
Content-Type: text/plain

{"error":"rate_limit_exceeded","message":"Too many requests"}

Configuration Examples

Tight Rate Limit (Anti-abuse):

// 1 request per second, burst of 3
rateLimiter := middleware.NewRateLimiter(1, 3)

Moderate Rate Limit (Standard API):

// 100 requests per second, burst of 20
rateLimiter := middleware.NewRateLimiter(100, 20)

Generous Rate Limit (Internal Services):

// 1000 requests per second, burst of 100
rateLimiter := middleware.NewRateLimiter(1000, 100)

Time-based Limits:

// 60 requests per minute = 1 request per second
rateLimiter := middleware.NewRateLimiter(1, 10)

// 1000 requests per hour ≈ 0.28 requests per second
rateLimiter := middleware.NewRateLimiter(0.28, 50)

Understanding Burst

The burst parameter allows short bursts above the rate:

// Rate: 10 rps, Burst: 5
// Allows up to 5 requests immediately, then 10/second
rateLimiter := middleware.NewRateLimiter(10, 5)

Bucket fills at rate: 10 tokens/second Bucket capacity: 5 tokens Request consumes: 1 token

Example traffic pattern:

T=0s: 5 requests → ✅ All allowed (burst)
T=0.1s: 1 request → ❌ Denied (bucket empty)
T=0.5s: 1 request → ✅ Allowed (bucket refilled 0.5 tokens)
T=1s: 1 request → ✅ Allowed (bucket has ~1 token)

Cleanup Behavior

The rate limiter automatically cleans up inactive limiters every 5 minutes to prevent memory leaks.

Performance Characteristics

Memory: ~100 bytes per active limiter
Throughput: >1M requests/second
Latency: <1μs per request
Concurrency: Lock-free for rate checks

Production Deployment

With Reverse Proxy:

// Use X-Forwarded-For or X-Real-IP
keyFunc := func(r *http.Request) string {
    // Check proxy headers first
    if ip := r.Header.Get("X-Forwarded-For"); ip != "" {
        return strings.Split(ip, ",")[0]
    }
    if ip := r.Header.Get("X-Real-IP"); ip != "" {
        return ip
    }
    return r.RemoteAddr
}

router.Use(rateLimiter.MiddlewareWithKeyFunc(keyFunc))

Environment-based Configuration:

import "os"

func getRateLimiter() *middleware.RateLimiter {
    rps := getEnvFloat("RATE_LIMIT_RPS", 100)
    burst := getEnvInt("RATE_LIMIT_BURST", 20)
    return middleware.NewRateLimiter(rps, burst)
}

Testing Rate Limits

# Send 10 requests rapidly
for i in {1..10}; do
  curl -w "Status: %{http_code}\n" http://localhost:8080/api/data
done

Expected output:

Status: 200  # Request 1-5 (within burst)
Status: 200
Status: 200
Status: 200
Status: 200
Status: 429  # Request 6-10 (rate limited)
Status: 429
Status: 429
Status: 429
Status: 429

Complete Example

package main

import (
    "encoding/json"
    "log"
    "net/http"
    "os"
    "strconv"

    "github.com/bitechdev/ResolveSpec/pkg/middleware"
    "github.com/gorilla/mux"
)

func main() {
    // Configuration from environment
    rps, _ := strconv.ParseFloat(os.Getenv("RATE_LIMIT_RPS"), 64)
    if rps == 0 {
        rps = 100 // Default
    }

    burst, _ := strconv.Atoi(os.Getenv("RATE_LIMIT_BURST"))
    if burst == 0 {
        burst = 20 // Default
    }

    // Create rate limiter
    rateLimiter := middleware.NewRateLimiter(rps, burst)

    // Custom key extraction
    keyFunc := func(r *http.Request) string {
        // Try API key first
        if apiKey := r.Header.Get("X-API-Key"); apiKey != "" {
            return "api:" + apiKey
        }
        // Try authenticated user
        if userID := r.Header.Get("X-User-ID"); userID != "" {
            return "user:" + userID
        }
        // Fall back to IP
        if ip := r.Header.Get("X-Forwarded-For"); ip != "" {
            return ip
        }
        return r.RemoteAddr
    }

    // Create router
    router := mux.NewRouter()

    // Apply rate limiting
    router.Use(rateLimiter.MiddlewareWithKeyFunc(keyFunc))

    // Routes
    router.HandleFunc("/api/data", dataHandler)
    router.HandleFunc("/health", healthHandler)

    log.Printf("Starting server with rate limit: %.1f rps, burst: %d", rps, burst)
    log.Fatal(http.ListenAndServe(":8080", router))
}

func dataHandler(w http.ResponseWriter, r *http.Request) {
    json.NewEncoder(w).Encode(map[string]string{
        "message": "Data endpoint",
    })
}

func healthHandler(w http.ResponseWriter, r *http.Request) {
    w.WriteHeader(http.StatusOK)
    w.Write([]byte("OK"))
}

Best Practices

Set Appropriate Limits: Consider your backend capacity
- Database: Can it handle X queries/second?
- External APIs: What are their rate limits?
- Server resources: CPU, memory, connections
Use Burst Wisely: Allow legitimate traffic spikes
- Too low: Reject valid bursts
- Too high: Allow abuse

Monitor Rate Limits: Track how often limits are hit

// Log rate limit events
if rateLimited {
    log.Printf("Rate limited: %s", clientKey)
}

Provide Feedback: Include rate limit headers (future enhancement)

X-RateLimit-Limit: 100
X-RateLimit-Remaining: 95
X-RateLimit-Reset: 1640000000

Tiered Limits: Different limits for different user tiers

func getRateLimiter(userTier string) *middleware.RateLimiter {
    switch userTier {
    case "premium":
        return middleware.NewRateLimiter(1000, 100)
    case "standard":
        return middleware.NewRateLimiter(100, 20)
    default:
        return middleware.NewRateLimiter(10, 5)
    }
}

Request Size Limits

Protect against oversized request bodies with configurable size limits.

Quick Start

import "github.com/bitechdev/ResolveSpec/pkg/middleware"

// Default: 10MB limit
sizeLimiter := middleware.NewRequestSizeLimiter(0)
router.Use(sizeLimiter.Middleware)

Custom Size Limit

// 5MB limit
sizeLimiter := middleware.NewRequestSizeLimiter(5 * 1024 * 1024)
router.Use(sizeLimiter.Middleware)

// Or use constants
sizeLimiter := middleware.NewRequestSizeLimiter(middleware.Size5MB)

Available Size Constants

middleware.Size1MB    // 1 MB
middleware.Size5MB    // 5 MB
middleware.Size10MB   // 10 MB (default)
middleware.Size50MB   // 50 MB
middleware.Size100MB  // 100 MB

Different Limits Per Route

func main() {
    router := mux.NewRouter()

    // File upload endpoint: 50MB
    uploadLimiter := middleware.NewRequestSizeLimiter(middleware.Size50MB)
    uploadRouter := router.PathPrefix("/upload").Subrouter()
    uploadRouter.Use(uploadLimiter.Middleware)

    // API endpoints: 1MB
    apiLimiter := middleware.NewRequestSizeLimiter(middleware.Size1MB)
    apiRouter := router.PathPrefix("/api").Subrouter()
    apiRouter.Use(apiLimiter.Middleware)
}

Dynamic Size Limits

// Custom size based on request
sizeFunc := func(r *http.Request) int64 {
    // Premium users get 50MB
    if isPremiumUser(r) {
        return middleware.Size50MB
    }
    // Free users get 5MB
    return middleware.Size5MB
}

router.Use(sizeLimiter.MiddlewareWithCustomSize(sizeFunc))

By Content-Type:

sizeFunc := func(r *http.Request) int64 {
    contentType := r.Header.Get("Content-Type")

    switch {
    case strings.Contains(contentType, "multipart/form-data"):
        return middleware.Size50MB // File uploads
    case strings.Contains(contentType, "application/json"):
        return middleware.Size1MB // JSON APIs
    default:
        return middleware.Size10MB // Default
    }
}

Error Response

When size limit exceeded:

HTTP/1.1 413 Request Entity Too Large
X-Max-Request-Size: 10485760

http: request body too large

Complete Example

package main

import (
    "log"
    "net/http"

    "github.com/bitechdev/ResolveSpec/pkg/middleware"
    "github.com/gorilla/mux"
)

func main() {
    router := mux.NewRouter()

    // API routes: 1MB limit
    api := router.PathPrefix("/api").Subrouter()
    apiLimiter := middleware.NewRequestSizeLimiter(middleware.Size1MB)
    api.Use(apiLimiter.Middleware)
    api.HandleFunc("/users", createUserHandler).Methods("POST")

    // Upload routes: 50MB limit
    upload := router.PathPrefix("/upload").Subrouter()
    uploadLimiter := middleware.NewRequestSizeLimiter(middleware.Size50MB)
    upload.Use(uploadLimiter.Middleware)
    upload.HandleFunc("/file", uploadFileHandler).Methods("POST")

    log.Fatal(http.ListenAndServe(":8080", router))
}

Input Sanitization

Protect against XSS, injection attacks, and malicious input.

Quick Start

import "github.com/bitechdev/ResolveSpec/pkg/middleware"

// Default sanitizer (safe defaults)
sanitizer := middleware.DefaultSanitizer()
router.Use(sanitizer.Middleware)

Sanitizer Types

Default Sanitizer (Recommended):

sanitizer := middleware.DefaultSanitizer()
// ✓ Escapes HTML entities
// ✓ Removes null bytes
// ✓ Removes control characters
// ✓ Blocks XSS patterns (script tags, event handlers)
// ✗ Does not strip HTML (allows legitimate content)

Strict Sanitizer:

sanitizer := middleware.StrictSanitizer()
// ✓ All default features
// ✓ Strips ALL HTML tags
// ✓ Max string length: 10,000 chars

Custom Configuration

sanitizer := &middleware.Sanitizer{
    StripHTML:          true,  // Remove HTML tags
    EscapeHTML:         false, // Don't escape (already stripped)
    RemoveNullBytes:    true,  // Remove \x00
    RemoveControlChars: true,  // Remove dangerous control chars
    MaxStringLength:    5000,  // Limit to 5000 chars

    // Block patterns (regex)
    BlockPatterns: []*regexp.Regexp{
        regexp.MustCompile(`(?i)<script`),
        regexp.MustCompile(`(?i)javascript:`),
    },

    // Custom sanitization function
    CustomSanitizer: func(s string) string {
        // Your custom logic
        return strings.ToLower(s)
    },
}

router.Use(sanitizer.Middleware)

What Gets Sanitized

Automatic (via middleware):

Query parameters
Headers (User-Agent, Referer, X-Forwarded-For, X-Real-IP)

Manual (in your handler):

Request body (JSON, form data)
Database queries
File names

Manual Sanitization

String Values:

sanitizer := middleware.DefaultSanitizer()

// Sanitize user input
username := sanitizer.Sanitize(r.FormValue("username"))
email := sanitizer.Sanitize(r.FormValue("email"))

Map/JSON Data:

var data map[string]interface{}
json.Unmarshal(body, &data)

// Sanitize all string values recursively
sanitizedData := sanitizer.SanitizeMap(data)

Nested Structures:

type User struct {
    Name    string
    Email   string
    Bio     string
    Profile map[string]interface{}
}

// After unmarshaling
user.Name = sanitizer.Sanitize(user.Name)
user.Email = sanitizer.Sanitize(user.Email)
user.Bio = sanitizer.Sanitize(user.Bio)
user.Profile = sanitizer.SanitizeMap(user.Profile)

Specialized Sanitizers

Filenames:

import "github.com/bitechdev/ResolveSpec/pkg/middleware"

filename := middleware.SanitizeFilename(uploadedFilename)
// Removes: .., /, \, null bytes
// Limits: 255 characters

Emails:

email := middleware.SanitizeEmail(" USER@EXAMPLE.COM ")
// Result: "user@example.com"
// Trims, lowercases, removes null bytes

URLs:

url := middleware.SanitizeURL(userInput)
// Blocks: javascript:, data: protocols
// Removes: null bytes

Blocked Patterns (Default)

The default sanitizer blocks:

Script tags: <script>...</script>
JavaScript protocol: javascript:alert(1)
Event handlers: onclick="...", onerror="..."
Iframes: <iframe src="...">
Objects: <object data="...">
Embeds: <embed src="...">

Security Best Practices

1. Layer Defense:

// Layer 1: Middleware (query params, headers)
router.Use(sanitizer.Middleware)

// Layer 2: Input validation (in handler)
func createUserHandler(w http.ResponseWriter, r *http.Request) {
    var user User
    json.NewDecoder(r.Body).Decode(&user)

    // Sanitize
    user.Name = sanitizer.Sanitize(user.Name)
    user.Email = middleware.SanitizeEmail(user.Email)

    // Validate
    if !isValidEmail(user.Email) {
        http.Error(w, "Invalid email", 400)
        return
    }

    // Use parameterized queries (prevents SQL injection)
    db.Exec("INSERT INTO users (name, email) VALUES (?, ?)",
        user.Name, user.Email)
}

2. Context-Aware Sanitization:

// HTML content (user posts, comments)
sanitizer := middleware.StrictSanitizer()
post.Content = sanitizer.Sanitize(post.Content)

// Structured data (JSON API)
sanitizer := middleware.DefaultSanitizer()
data = sanitizer.SanitizeMap(jsonData)

// Search queries (preserve special chars)
query = middleware.SanitizeFilename(searchTerm) // Light sanitization

3. Output Encoding:

// When rendering HTML
import "html/template"

tmpl := template.Must(template.New("page").Parse(`
    <h1>{{.Title}}</h1>
    <p>{{.Content}}</p>
`))

// template.HTML automatically escapes
tmpl.Execute(w, data)

Complete Example

package main

import (
    "encoding/json"
    "log"
    "net/http"

    "github.com/bitechdev/ResolveSpec/pkg/middleware"
    "github.com/gorilla/mux"
)

func main() {
    router := mux.NewRouter()

    // Apply sanitization middleware
    sanitizer := middleware.DefaultSanitizer()
    router.Use(sanitizer.Middleware)

    router.HandleFunc("/api/users", createUserHandler).Methods("POST")

    log.Fatal(http.ListenAndServe(":8080", router))
}

func createUserHandler(w http.ResponseWriter, r *http.Request) {
    sanitizer := middleware.DefaultSanitizer()

    var user struct {
        Name  string `json:"name"`
        Email string `json:"email"`
        Bio   string `json:"bio"`
    }

    if err := json.NewDecoder(r.Body).Decode(&user); err != nil {
        http.Error(w, "Invalid JSON", 400)
        return
    }

    // Sanitize inputs
    user.Name = sanitizer.Sanitize(user.Name)
    user.Email = middleware.SanitizeEmail(user.Email)
    user.Bio = sanitizer.Sanitize(user.Bio)

    // Validate
    if len(user.Name) == 0 || len(user.Email) == 0 {
        http.Error(w, "Name and email required", 400)
        return
    }

    // Save to database (use parameterized queries!)
    // db.Exec("INSERT INTO users (name, email, bio) VALUES (?, ?, ?)",
    //     user.Name, user.Email, user.Bio)

    w.WriteHeader(http.StatusCreated)
    json.NewEncoder(w).Encode(map[string]string{
        "status": "created",
    })
}

Testing Sanitization

# Test XSS prevention
curl -X POST http://localhost:8080/api/users \
  -H "Content-Type: application/json" \
  -d '{
    "name": "<script>alert(1)</script>John",
    "email": "test@example.com",
    "bio": "My bio with <iframe src=\"evil.com\"></iframe>"
  }'

# Script tags and iframes should be removed

Performance

Overhead: <1ms per request for typical payloads
Regex compilation: Done once at initialization
Safe for production: Minimal performance impact

18 KiB Raw Blame History

Middleware Package

Table of Contents

Rate Limiting

Quick Start

Basic Usage

Custom Key Extraction

Advanced Key Functions

Different Limits Per Route

Rate Limit Response

Configuration Examples

Understanding Burst

Cleanup Behavior

Performance Characteristics

Production Deployment

Testing Rate Limits

Complete Example

Best Practices

Request Size Limits

Quick Start

Custom Size Limit

Available Size Constants

Different Limits Per Route

Dynamic Size Limits

Error Response

Complete Example

Input Sanitization

Quick Start

Sanitizer Types

Custom Configuration

What Gets Sanitized

Manual Sanitization

Specialized Sanitizers

Blocked Patterns (Default)

Security Best Practices

Complete Example

Testing Sanitization

Performance

18 KiB

Raw Blame History