feat: implement file upload handler and related functionality
- Added file upload handler to process both multipart and raw file uploads. - Implemented parsing logic for upload requests, including handling file metadata. - Introduced SaveFileDecodedInput structure for handling decoded file uploads. - Created unit tests for file upload parsing and validation. feat: add metadata retry configuration and functionality - Introduced MetadataRetryConfig to the application configuration. - Implemented MetadataRetryer to handle retrying metadata extraction for thoughts. - Added new tool for retrying failed metadata extractions. - Updated thought metadata structure to include status and timestamps for metadata processing. fix: enhance metadata normalization and error handling - Updated metadata normalization functions to track status and errors. - Improved handling of metadata extraction failures during thought updates and captures. - Ensured that metadata status is correctly set during various operations. refactor: streamline file saving logic in FilesTool - Refactored Save method in FilesTool to utilize new SaveDecoded method. - Simplified project and thought ID resolution logic during file saving.
This commit is contained in:
@@ -93,6 +93,25 @@ func Run(ctx context.Context, configPath string) error {
|
||||
}()
|
||||
}
|
||||
|
||||
if cfg.MetadataRetry.Enabled && cfg.MetadataRetry.RunOnStartup {
|
||||
go runMetadataRetryPass(ctx, db, provider, cfg, activeProjects, logger)
|
||||
}
|
||||
|
||||
if cfg.MetadataRetry.Enabled && cfg.MetadataRetry.Interval > 0 {
|
||||
go func() {
|
||||
ticker := time.NewTicker(cfg.MetadataRetry.Interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
runMetadataRetryPass(ctx, db, provider, cfg, activeProjects, logger)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port),
|
||||
Handler: routes(logger, cfg, db, provider, keyring, oauthRegistry, tokenStore, authCodes, dynClients, activeProjects),
|
||||
@@ -127,33 +146,38 @@ func Run(ctx context.Context, configPath string) error {
|
||||
|
||||
func routes(logger *slog.Logger, cfg *config.Config, db *store.DB, provider ai.Provider, keyring *auth.Keyring, oauthRegistry *auth.OAuthRegistry, tokenStore *auth.TokenStore, authCodes *auth.AuthCodeStore, dynClients *auth.DynamicClientStore, activeProjects *session.ActiveProjects) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
authMiddleware := auth.Middleware(cfg.Auth, keyring, oauthRegistry, tokenStore, logger)
|
||||
filesTool := tools.NewFilesTool(db, activeProjects)
|
||||
metadataRetryer := tools.NewMetadataRetryer(context.Background(), db, provider, cfg.Capture, cfg.AI.Metadata.Timeout, activeProjects, logger)
|
||||
|
||||
toolSet := mcpserver.ToolSet{
|
||||
Capture: tools.NewCaptureTool(db, provider, cfg.Capture, cfg.AI.Metadata.Timeout, activeProjects, logger),
|
||||
Search: tools.NewSearchTool(db, provider, cfg.Search, activeProjects),
|
||||
List: tools.NewListTool(db, cfg.Search, activeProjects),
|
||||
Stats: tools.NewStatsTool(db),
|
||||
Get: tools.NewGetTool(db),
|
||||
Update: tools.NewUpdateTool(db, provider, cfg.Capture, logger),
|
||||
Delete: tools.NewDeleteTool(db),
|
||||
Archive: tools.NewArchiveTool(db),
|
||||
Projects: tools.NewProjectsTool(db, activeProjects),
|
||||
Context: tools.NewContextTool(db, provider, cfg.Search, activeProjects),
|
||||
Recall: tools.NewRecallTool(db, provider, cfg.Search, activeProjects),
|
||||
Summarize: tools.NewSummarizeTool(db, provider, cfg.Search, activeProjects),
|
||||
Links: tools.NewLinksTool(db, provider, cfg.Search),
|
||||
Files: tools.NewFilesTool(db, activeProjects),
|
||||
Backfill: tools.NewBackfillTool(db, provider, activeProjects, logger),
|
||||
Reparse: tools.NewReparseMetadataTool(db, provider, cfg.Capture, activeProjects, logger),
|
||||
Household: tools.NewHouseholdTool(db),
|
||||
Maintenance: tools.NewMaintenanceTool(db),
|
||||
Calendar: tools.NewCalendarTool(db),
|
||||
Meals: tools.NewMealsTool(db),
|
||||
CRM: tools.NewCRMTool(db),
|
||||
Capture: tools.NewCaptureTool(db, provider, cfg.Capture, cfg.AI.Metadata.Timeout, activeProjects, metadataRetryer, logger),
|
||||
Search: tools.NewSearchTool(db, provider, cfg.Search, activeProjects),
|
||||
List: tools.NewListTool(db, cfg.Search, activeProjects),
|
||||
Stats: tools.NewStatsTool(db),
|
||||
Get: tools.NewGetTool(db),
|
||||
Update: tools.NewUpdateTool(db, provider, cfg.Capture, logger),
|
||||
Delete: tools.NewDeleteTool(db),
|
||||
Archive: tools.NewArchiveTool(db),
|
||||
Projects: tools.NewProjectsTool(db, activeProjects),
|
||||
Context: tools.NewContextTool(db, provider, cfg.Search, activeProjects),
|
||||
Recall: tools.NewRecallTool(db, provider, cfg.Search, activeProjects),
|
||||
Summarize: tools.NewSummarizeTool(db, provider, cfg.Search, activeProjects),
|
||||
Links: tools.NewLinksTool(db, provider, cfg.Search),
|
||||
Files: filesTool,
|
||||
Backfill: tools.NewBackfillTool(db, provider, activeProjects, logger),
|
||||
Reparse: tools.NewReparseMetadataTool(db, provider, cfg.Capture, activeProjects, logger),
|
||||
RetryMetadata: tools.NewRetryMetadataTool(metadataRetryer),
|
||||
Household: tools.NewHouseholdTool(db),
|
||||
Maintenance: tools.NewMaintenanceTool(db),
|
||||
Calendar: tools.NewCalendarTool(db),
|
||||
Meals: tools.NewMealsTool(db),
|
||||
CRM: tools.NewCRMTool(db),
|
||||
}
|
||||
|
||||
mcpHandler := mcpserver.New(cfg.MCP, toolSet)
|
||||
mux.Handle(cfg.MCP.Path, auth.Middleware(cfg.Auth, keyring, oauthRegistry, tokenStore, logger)(mcpHandler))
|
||||
mux.Handle(cfg.MCP.Path, authMiddleware(mcpHandler))
|
||||
mux.Handle("/files", authMiddleware(fileUploadHandler(filesTool)))
|
||||
if oauthRegistry != nil && tokenStore != nil {
|
||||
mux.HandleFunc("/.well-known/oauth-authorization-server", oauthMetadataHandler())
|
||||
mux.HandleFunc("/oauth-authorization-server", oauthMetadataHandler())
|
||||
@@ -245,6 +269,25 @@ func routes(logger *slog.Logger, cfg *config.Config, db *store.DB, provider ai.P
|
||||
)
|
||||
}
|
||||
|
||||
func runMetadataRetryPass(ctx context.Context, db *store.DB, provider ai.Provider, cfg *config.Config, activeProjects *session.ActiveProjects, logger *slog.Logger) {
|
||||
retryer := tools.NewMetadataRetryer(ctx, db, provider, cfg.Capture, cfg.AI.Metadata.Timeout, activeProjects, logger)
|
||||
_, out, err := retryer.Handle(ctx, nil, tools.RetryMetadataInput{
|
||||
Limit: cfg.MetadataRetry.MaxPerRun,
|
||||
IncludeArchived: cfg.MetadataRetry.IncludeArchived,
|
||||
OlderThanDays: 1,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("auto metadata retry failed", slog.String("error", err.Error()))
|
||||
return
|
||||
}
|
||||
logger.Info("auto metadata retry pass",
|
||||
slog.Int("scanned", out.Scanned),
|
||||
slog.Int("retried", out.Retried),
|
||||
slog.Int("updated", out.Updated),
|
||||
slog.Int("failed", out.Failed),
|
||||
)
|
||||
}
|
||||
|
||||
func runBackfillPass(ctx context.Context, db *store.DB, provider ai.Provider, cfg config.BackfillConfig, logger *slog.Logger) {
|
||||
backfiller := tools.NewBackfillTool(db, provider, nil, logger)
|
||||
_, out, err := backfiller.Handle(ctx, nil, tools.BackfillInput{
|
||||
|
||||
Reference in New Issue
Block a user