Fixed filterExtendedOptions

go.mod

@@ -5,12 +5,15 @@ go 1.24.0
 toolchain go1.24.6
 
 require (
+	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/bradfitz/gomemcache v0.0.0-20250403215159-8d39553ac7cf
+	github.com/getsentry/sentry-go v0.40.0
 	github.com/glebarez/sqlite v1.11.0
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.1
 	github.com/prometheus/client_golang v1.23.2
 	github.com/redis/go-redis/v9 v9.17.1
+	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
 	github.com/tidwall/gjson v1.18.0
 	github.com/tidwall/sjson v1.2.5
@@ -30,7 +33,6 @@ require (
 )
 
 require (
-	github.com/DATA-DOG/go-sqlmock v1.5.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
@@ -38,7 +40,6 @@ require (
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/getsentry/sentry-go v0.40.0 // indirect
 	github.com/glebarez/go-sqlite v1.21.2 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -66,7 +67,6 @@ require (
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
-	github.com/spf13/viper v1.21.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect

go.sum

@@ -19,6 +19,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/getsentry/sentry-go v0.40.0 h1:VTJMN9zbTvqDqPwheRVLcp0qcUcM+8eFivvGocAaSbo=
@@ -27,6 +29,8 @@ github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9g
 github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
 github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
 github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -77,6 +81,10 @@ github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdh
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=

pkg/common/adapters/database/bun.go

@@ -34,6 +34,63 @@ func (h *QueryDebugHook) AfterQuery(ctx context.Context, event *bun.QueryEvent)
 	}
 }
 
+// debugScanIntoStruct attempts to scan rows into a struct with detailed field-level logging
+// This helps identify which specific field is causing scanning issues
+func debugScanIntoStruct(rows interface{}, dest interface{}) error {
+	v := reflect.ValueOf(dest)
+	if v.Kind() != reflect.Ptr {
+		return fmt.Errorf("dest must be a pointer")
+	}
+
+	v = v.Elem()
+	if v.Kind() != reflect.Struct && v.Kind() != reflect.Slice {
+		return fmt.Errorf("dest must be pointer to struct or slice")
+	}
+
+	// Log the type being scanned into
+	typeName := v.Type().String()
+	logger.Debug("Debug scan into type: %s (kind: %s)", typeName, v.Kind())
+
+	// Handle slice types - inspect the element type
+	var structType reflect.Type
+	if v.Kind() == reflect.Slice {
+		elemType := v.Type().Elem()
+		logger.Debug("  Slice element type: %s", elemType)
+
+		// If slice of pointers, get the underlying type
+		if elemType.Kind() == reflect.Ptr {
+			structType = elemType.Elem()
+		} else {
+			structType = elemType
+		}
+	} else if v.Kind() == reflect.Struct {
+		structType = v.Type()
+	}
+
+	// If we have a struct type, log all its fields
+	if structType != nil && structType.Kind() == reflect.Struct {
+		logger.Debug("  Struct %s has %d fields:", structType.Name(), structType.NumField())
+		for i := 0; i < structType.NumField(); i++ {
+			field := structType.Field(i)
+
+			// Log embedded fields specially
+			if field.Anonymous {
+				logger.Debug("    [%d] EMBEDDED: %s (type: %s, kind: %s, bun:%q)",
+					i, field.Name, field.Type, field.Type.Kind(), field.Tag.Get("bun"))
+			} else {
+				bunTag := field.Tag.Get("bun")
+				if bunTag == "" {
+					bunTag = "(no tag)"
+				}
+				logger.Debug("    [%d] %s (type: %s, kind: %s, bun:%q)",
+					i, field.Name, field.Type, field.Type.Kind(), bunTag)
+			}
+		}
+	}
+
+	return nil
+}
+
 // BunAdapter adapts Bun to work with our Database interface
 // This demonstrates how the abstraction works with different ORMs
 type BunAdapter struct {
@@ -52,6 +109,14 @@ func (b *BunAdapter) EnableQueryDebug() {
 	logger.Info("Bun query debug mode enabled - all SQL queries will be logged")
 }
 
+// EnableDetailedScanDebug enables verbose logging of scan operations
+// WARNING: This generates a LOT of log output. Use only for debugging specific issues.
+func (b *BunAdapter) EnableDetailedScanDebug() {
+	logger.Info("Detailed scan debugging enabled - will log all field scanning operations")
+	// This is a flag that can be checked in scan operations
+	// Implementation would require modifying the scan logic
+}
+
 // DisableQueryDebug removes all query hooks
 func (b *BunAdapter) DisableQueryDebug() {
 	// Create a new DB without hooks
@@ -676,6 +741,31 @@ func (b *BunSelectQuery) Scan(ctx context.Context, dest interface{}) (err error)
 func (b *BunSelectQuery) ScanModel(ctx context.Context) (err error) {
 	defer func() {
 		if r := recover(); r != nil {
+			// Enhanced panic recovery with model information
+			model := b.query.GetModel()
+			var modelInfo string
+			if model != nil && model.Value() != nil {
+				modelValue := model.Value()
+				modelInfo = fmt.Sprintf("Model type: %T", modelValue)
+
+				// Try to get the model's underlying struct type
+				v := reflect.ValueOf(modelValue)
+				if v.Kind() == reflect.Ptr {
+					v = v.Elem()
+				}
+				if v.Kind() == reflect.Slice {
+					if v.Type().Elem().Kind() == reflect.Ptr {
+						modelInfo += fmt.Sprintf(", Slice of: %s", v.Type().Elem().Elem().Name())
+					} else {
+						modelInfo += fmt.Sprintf(", Slice of: %s", v.Type().Elem().Name())
+					}
+				} else if v.Kind() == reflect.Struct {
+					modelInfo += fmt.Sprintf(", Struct: %s", v.Type().Name())
+				}
+			}
+
+			sqlStr := b.query.String()
+			logger.Error("Panic in BunSelectQuery.ScanModel: %v. %s. SQL: %s", r, modelInfo, sqlStr)
 			err = logger.HandlePanic("BunSelectQuery.ScanModel", r)
 		}
 	}()
@@ -683,6 +773,17 @@ func (b *BunSelectQuery) ScanModel(ctx context.Context) (err error) {
 		return fmt.Errorf("model is nil")
 	}
 
+	// Optional: Enable detailed field-level debugging (set to true to debug)
+	const enableDetailedDebug = true
+	if enableDetailedDebug {
+		model := b.query.GetModel()
+		if model != nil && model.Value() != nil {
+			if err := debugScanIntoStruct(nil, model.Value()); err != nil {
+				logger.Warn("Debug scan inspection failed: %v", err)
+			}
+		}
+	}
+
 	// Execute the main query first
 	err = b.query.Scan(ctx)
 	if err != nil {

pkg/common/sql_helpers.go

@@ -393,6 +393,7 @@ func getValidColumnsForTable(tableName string) map[string]bool {
 // extractTableAndColumn extracts the table prefix and column name from a qualified reference
 // For example: "users.status = 'active'" returns ("users", "status")
 // Returns empty strings if no table prefix is found
+// This function is parenthesis-aware and will only look for operators outside of subqueries
 func extractTableAndColumn(cond string) (table string, column string) {
 	// Common SQL operators to find the column reference
 	operators := []string{" = ", " != ", " <> ", " > ", " >= ", " < ", " <= ", " LIKE ", " like ", " IN ", " in ", " IS ", " is "}
@@ -400,13 +401,20 @@ func extractTableAndColumn(cond string) (table string, column string) {
 	var columnRef string
 
 	// Find the column reference (left side of the operator)
+	// We need to find the first operator that appears OUTSIDE of parentheses
+	minIdx := -1
+
 	for _, op := range operators {
-		if idx := strings.Index(cond, op); idx > 0 {
-			columnRef = strings.TrimSpace(cond[:idx])
-			break
+		idx := findOperatorOutsideParentheses(cond, op)
+		if idx > 0 && (minIdx == -1 || idx < minIdx) {
+			minIdx = idx
 		}
 	}
 
+	if minIdx > 0 {
+		columnRef = strings.TrimSpace(cond[:minIdx])
+	}
+
 	// If no operator found, the whole condition might be the column reference
 	if columnRef == "" {
 		parts := strings.Fields(cond)
@@ -422,7 +430,45 @@ func extractTableAndColumn(cond string) (table string, column string) {
 	// Remove any quotes
 	columnRef = strings.Trim(columnRef, "`\"'")
 
-	// Check if it contains a dot (qualified reference)
+	// Check if there's a function call (contains opening parenthesis)
+	openParenIdx := strings.Index(columnRef, "(")
+
+	if openParenIdx >= 0 {
+		// There's a function call - find the FIRST dot after the opening paren
+		// This handles cases like: ifblnk(users.status, orders.status) - extracts users.status
+		dotIdx := strings.Index(columnRef[openParenIdx:], ".")
+		if dotIdx > 0 {
+			dotIdx += openParenIdx // Adjust to absolute position
+
+			// Extract table name (between paren and dot)
+			// Find the last opening paren before this dot
+			lastOpenParen := strings.LastIndex(columnRef[:dotIdx], "(")
+			table = columnRef[lastOpenParen+1 : dotIdx]
+
+			// Find the column name - it ends at comma, closing paren, whitespace, or end of string
+			columnStart := dotIdx + 1
+			columnEnd := len(columnRef)
+
+			for i := columnStart; i < len(columnRef); i++ {
+				ch := columnRef[i]
+				if ch == ',' || ch == ')' || ch == ' ' || ch == '\t' {
+					columnEnd = i
+					break
+				}
+			}
+
+			column = columnRef[columnStart:columnEnd]
+
+			// Remove quotes from table and column if present
+			table = strings.Trim(table, "`\"'")
+			column = strings.Trim(column, "`\"'")
+
+			return table, column
+		}
+	}
+
+	// No function call - check if it contains a dot (qualified reference)
+	// Use LastIndex to handle schema.table.column properly
 	if dotIdx := strings.LastIndex(columnRef, "."); dotIdx > 0 {
 		table = columnRef[:dotIdx]
 		column = columnRef[dotIdx+1:]
@@ -437,6 +483,53 @@ func extractTableAndColumn(cond string) (table string, column string) {
 	return "", ""
 }
 
+// findOperatorOutsideParentheses finds the first occurrence of an operator outside of parentheses
+// Returns the index of the operator, or -1 if not found or only found inside parentheses
+func findOperatorOutsideParentheses(s string, operator string) int {
+	depth := 0
+	inSingleQuote := false
+	inDoubleQuote := false
+
+	for i := 0; i < len(s); i++ {
+		ch := s[i]
+
+		// Track quote state (operators inside quotes should be ignored)
+		if ch == '\'' && !inDoubleQuote {
+			inSingleQuote = !inSingleQuote
+			continue
+		}
+		if ch == '"' && !inSingleQuote {
+			inDoubleQuote = !inDoubleQuote
+			continue
+		}
+
+		// Skip if we're inside quotes
+		if inSingleQuote || inDoubleQuote {
+			continue
+		}
+
+		// Track parenthesis depth
+		switch ch {
+		case '(':
+			depth++
+		case ')':
+			depth--
+		}
+
+		// Only look for the operator when we're outside parentheses (depth == 0)
+		if depth == 0 {
+			// Check if the operator starts at this position
+			if i+len(operator) <= len(s) {
+				if s[i:i+len(operator)] == operator {
+					return i
+				}
+			}
+		}
+	}
+
+	return -1
+}
+
 // isValidColumn checks if a column name exists in the valid columns map
 // Handles case-insensitive comparison
 func isValidColumn(columnName string, validColumns map[string]bool) bool {

pkg/common/sql_helpers_test.go

@@ -122,6 +122,18 @@ func TestSanitizeWhereClause(t *testing.T) {
 			tableName: "users",
 			expected:  "",
 		},
+		{
+			name:      "subquery with table alias should not be modified",
+			where:     "apiprovider.rid_apiprovider in (select l.rid_apiprovider from core.apiproviderlink l where l.rid_hub = 2576)",
+			tableName: "apiprovider",
+			expected:  "apiprovider.rid_apiprovider in (select l.rid_apiprovider from core.apiproviderlink l where l.rid_hub = 2576)",
+		},
+		{
+			name:      "complex subquery with AND and multiple operators",
+			where:     "apiprovider.type in ('softphone') AND (apiprovider.rid_apiprovider in (select l.rid_apiprovider from core.apiproviderlink l where l.rid_hub = 2576))",
+			tableName: "apiprovider",
+			expected:  "apiprovider.type in ('softphone') AND (apiprovider.rid_apiprovider in (select l.rid_apiprovider from core.apiproviderlink l where l.rid_hub = 2576))",
+		},
 	}
 
 	for _, tt := range tests {
@@ -274,6 +286,48 @@ func TestExtractTableAndColumn(t *testing.T) {
 			expectedTable: "",
 			expectedCol:   "",
 		},
+		{
+			name:          "function call with table.column - ifblnk",
+			input:         "ifblnk(users.status,0) in (1,2,3,4)",
+			expectedTable: "users",
+			expectedCol:   "status",
+		},
+		{
+			name:          "function call with table.column - coalesce",
+			input:         "coalesce(users.age, 0) = 25",
+			expectedTable: "users",
+			expectedCol:   "age",
+		},
+		{
+			name:          "nested function calls",
+			input:         "upper(trim(users.name)) = 'JOHN'",
+			expectedTable: "users",
+			expectedCol:   "name",
+		},
+		{
+			name:          "function with multiple args and table.column",
+			input:         "substring(users.email, 1, 5) = 'admin'",
+			expectedTable: "users",
+			expectedCol:   "email",
+		},
+		{
+			name:          "cast function with table.column",
+			input:         "cast(orders.total as decimal) > 100",
+			expectedTable: "orders",
+			expectedCol:   "total",
+		},
+		{
+			name:          "complex nested functions",
+			input:         "coalesce(nullif(users.status, ''), 'default') = 'active'",
+			expectedTable: "users",
+			expectedCol:   "status",
+		},
+		{
+			name:          "function with multiple table.column refs (extracts first)",
+			input:         "greatest(users.created_at, users.updated_at) > '2024-01-01'",
+			expectedTable: "users",
+			expectedCol:   "created_at",
+		},
 	}
 
 	for _, tt := range tests {
@@ -340,6 +394,14 @@ func TestSanitizeWhereClauseWithPreloads(t *testing.T) {
 			},
 			expected: "users.status = 'active' AND Department.name = 'Engineering'",
 		},
+
+		{
+			name:      "Function Call with correct table prefix - unchanged",
+			where:     "ifblnk(users.status,0) in (1,2,3,4)",
+			tableName: "users",
+			options:   nil,
+			expected:  "ifblnk(users.status,0) in (1,2,3,4)",
+		},
 		{
 			name:      "no options provided - works as before",
 			where:     "wrong_table.status = 'active'",

pkg/restheadspec/handler.go

@@ -127,7 +127,7 @@ func (h *Handler) Handle(w common.ResponseWriter, r common.Request, params map[s
 
 	// Validate and filter columns in options (log warnings for invalid columns)
 	validator := common.NewColumnValidator(model)
-	options = filterExtendedOptions(validator, options)
+	options = h.filterExtendedOptions(validator, options, model)
 
 	// Add request-scoped data to context (including options)
 	ctx = WithRequestData(ctx, schema, entity, tableName, model, modelPtr, options)
@@ -2241,7 +2241,7 @@ func (h *Handler) setRowNumbersOnRecords(records any, offset int) {
 }
 
 // filterExtendedOptions filters all column references, removing invalid ones and logging warnings
-func filterExtendedOptions(validator *common.ColumnValidator, options ExtendedRequestOptions) ExtendedRequestOptions {
+func (h *Handler) filterExtendedOptions(validator *common.ColumnValidator, options ExtendedRequestOptions, model interface{}) ExtendedRequestOptions {
 	filtered := options
 
 	// Filter base RequestOptions
@@ -2265,12 +2265,30 @@ func filterExtendedOptions(validator *common.ColumnValidator, options ExtendedRe
 	// No filtering needed for ComputedQL keys
 	filtered.ComputedQL = options.ComputedQL
 
-	// Filter Expand columns
+	// Filter Expand columns using the expand relation's model
 	filteredExpands := make([]ExpandOption, 0, len(options.Expand))
+	modelType := reflect.TypeOf(model)
+	if modelType.Kind() == reflect.Ptr {
+		modelType = modelType.Elem()
+	}
+
 	for _, expand := range options.Expand {
 		filteredExpand := expand
-		// Don't validate relation name, only columns
-		filteredExpand.Columns = validator.FilterValidColumns(expand.Columns)
+
+		// Get the relationship info for this expand relation
+		relInfo := h.getRelationshipInfo(modelType, expand.Relation)
+		if relInfo != nil && relInfo.relatedModel != nil {
+			// Create a validator for the related model
+			expandValidator := common.NewColumnValidator(relInfo.relatedModel)
+			// Filter columns using the related model's validator
+			filteredExpand.Columns = expandValidator.FilterValidColumns(expand.Columns)
+		} else {
+			// If we can't find the relationship, log a warning and skip column filtering
+			logger.Warn("Cannot validate columns for unknown relation: %s", expand.Relation)
+			// Keep the columns as-is if we can't validate them
+			filteredExpand.Columns = expand.Columns
+		}
+
 		filteredExpands = append(filteredExpands, filteredExpand)
 	}
 	filtered.Expand = filteredExpands