feat: implement file upload handler and related functionality
- Added file upload handler to process both multipart and raw file uploads. - Implemented parsing logic for upload requests, including handling file metadata. - Introduced SaveFileDecodedInput structure for handling decoded file uploads. - Created unit tests for file upload parsing and validation. feat: add metadata retry configuration and functionality - Introduced MetadataRetryConfig to the application configuration. - Implemented MetadataRetryer to handle retrying metadata extraction for thoughts. - Added new tool for retrying failed metadata extractions. - Updated thought metadata structure to include status and timestamps for metadata processing. fix: enhance metadata normalization and error handling - Updated metadata normalization functions to track status and errors. - Improved handling of metadata extraction failures during thought updates and captures. - Ensured that metadata status is correctly set during various operations. refactor: streamline file saving logic in FilesTool - Refactored Save method in FilesTool to utilize new SaveDecoded method. - Simplified project and thought ID resolution logic during file saving.
This commit is contained in:
@@ -93,6 +93,25 @@ func Run(ctx context.Context, configPath string) error {
|
||||
}()
|
||||
}
|
||||
|
||||
if cfg.MetadataRetry.Enabled && cfg.MetadataRetry.RunOnStartup {
|
||||
go runMetadataRetryPass(ctx, db, provider, cfg, activeProjects, logger)
|
||||
}
|
||||
|
||||
if cfg.MetadataRetry.Enabled && cfg.MetadataRetry.Interval > 0 {
|
||||
go func() {
|
||||
ticker := time.NewTicker(cfg.MetadataRetry.Interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
runMetadataRetryPass(ctx, db, provider, cfg, activeProjects, logger)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port),
|
||||
Handler: routes(logger, cfg, db, provider, keyring, oauthRegistry, tokenStore, authCodes, dynClients, activeProjects),
|
||||
@@ -127,33 +146,38 @@ func Run(ctx context.Context, configPath string) error {
|
||||
|
||||
func routes(logger *slog.Logger, cfg *config.Config, db *store.DB, provider ai.Provider, keyring *auth.Keyring, oauthRegistry *auth.OAuthRegistry, tokenStore *auth.TokenStore, authCodes *auth.AuthCodeStore, dynClients *auth.DynamicClientStore, activeProjects *session.ActiveProjects) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
authMiddleware := auth.Middleware(cfg.Auth, keyring, oauthRegistry, tokenStore, logger)
|
||||
filesTool := tools.NewFilesTool(db, activeProjects)
|
||||
metadataRetryer := tools.NewMetadataRetryer(context.Background(), db, provider, cfg.Capture, cfg.AI.Metadata.Timeout, activeProjects, logger)
|
||||
|
||||
toolSet := mcpserver.ToolSet{
|
||||
Capture: tools.NewCaptureTool(db, provider, cfg.Capture, cfg.AI.Metadata.Timeout, activeProjects, logger),
|
||||
Search: tools.NewSearchTool(db, provider, cfg.Search, activeProjects),
|
||||
List: tools.NewListTool(db, cfg.Search, activeProjects),
|
||||
Stats: tools.NewStatsTool(db),
|
||||
Get: tools.NewGetTool(db),
|
||||
Update: tools.NewUpdateTool(db, provider, cfg.Capture, logger),
|
||||
Delete: tools.NewDeleteTool(db),
|
||||
Archive: tools.NewArchiveTool(db),
|
||||
Projects: tools.NewProjectsTool(db, activeProjects),
|
||||
Context: tools.NewContextTool(db, provider, cfg.Search, activeProjects),
|
||||
Recall: tools.NewRecallTool(db, provider, cfg.Search, activeProjects),
|
||||
Summarize: tools.NewSummarizeTool(db, provider, cfg.Search, activeProjects),
|
||||
Links: tools.NewLinksTool(db, provider, cfg.Search),
|
||||
Files: tools.NewFilesTool(db, activeProjects),
|
||||
Backfill: tools.NewBackfillTool(db, provider, activeProjects, logger),
|
||||
Reparse: tools.NewReparseMetadataTool(db, provider, cfg.Capture, activeProjects, logger),
|
||||
Household: tools.NewHouseholdTool(db),
|
||||
Maintenance: tools.NewMaintenanceTool(db),
|
||||
Calendar: tools.NewCalendarTool(db),
|
||||
Meals: tools.NewMealsTool(db),
|
||||
CRM: tools.NewCRMTool(db),
|
||||
Capture: tools.NewCaptureTool(db, provider, cfg.Capture, cfg.AI.Metadata.Timeout, activeProjects, metadataRetryer, logger),
|
||||
Search: tools.NewSearchTool(db, provider, cfg.Search, activeProjects),
|
||||
List: tools.NewListTool(db, cfg.Search, activeProjects),
|
||||
Stats: tools.NewStatsTool(db),
|
||||
Get: tools.NewGetTool(db),
|
||||
Update: tools.NewUpdateTool(db, provider, cfg.Capture, logger),
|
||||
Delete: tools.NewDeleteTool(db),
|
||||
Archive: tools.NewArchiveTool(db),
|
||||
Projects: tools.NewProjectsTool(db, activeProjects),
|
||||
Context: tools.NewContextTool(db, provider, cfg.Search, activeProjects),
|
||||
Recall: tools.NewRecallTool(db, provider, cfg.Search, activeProjects),
|
||||
Summarize: tools.NewSummarizeTool(db, provider, cfg.Search, activeProjects),
|
||||
Links: tools.NewLinksTool(db, provider, cfg.Search),
|
||||
Files: filesTool,
|
||||
Backfill: tools.NewBackfillTool(db, provider, activeProjects, logger),
|
||||
Reparse: tools.NewReparseMetadataTool(db, provider, cfg.Capture, activeProjects, logger),
|
||||
RetryMetadata: tools.NewRetryMetadataTool(metadataRetryer),
|
||||
Household: tools.NewHouseholdTool(db),
|
||||
Maintenance: tools.NewMaintenanceTool(db),
|
||||
Calendar: tools.NewCalendarTool(db),
|
||||
Meals: tools.NewMealsTool(db),
|
||||
CRM: tools.NewCRMTool(db),
|
||||
}
|
||||
|
||||
mcpHandler := mcpserver.New(cfg.MCP, toolSet)
|
||||
mux.Handle(cfg.MCP.Path, auth.Middleware(cfg.Auth, keyring, oauthRegistry, tokenStore, logger)(mcpHandler))
|
||||
mux.Handle(cfg.MCP.Path, authMiddleware(mcpHandler))
|
||||
mux.Handle("/files", authMiddleware(fileUploadHandler(filesTool)))
|
||||
if oauthRegistry != nil && tokenStore != nil {
|
||||
mux.HandleFunc("/.well-known/oauth-authorization-server", oauthMetadataHandler())
|
||||
mux.HandleFunc("/oauth-authorization-server", oauthMetadataHandler())
|
||||
@@ -245,6 +269,25 @@ func routes(logger *slog.Logger, cfg *config.Config, db *store.DB, provider ai.P
|
||||
)
|
||||
}
|
||||
|
||||
func runMetadataRetryPass(ctx context.Context, db *store.DB, provider ai.Provider, cfg *config.Config, activeProjects *session.ActiveProjects, logger *slog.Logger) {
|
||||
retryer := tools.NewMetadataRetryer(ctx, db, provider, cfg.Capture, cfg.AI.Metadata.Timeout, activeProjects, logger)
|
||||
_, out, err := retryer.Handle(ctx, nil, tools.RetryMetadataInput{
|
||||
Limit: cfg.MetadataRetry.MaxPerRun,
|
||||
IncludeArchived: cfg.MetadataRetry.IncludeArchived,
|
||||
OlderThanDays: 1,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("auto metadata retry failed", slog.String("error", err.Error()))
|
||||
return
|
||||
}
|
||||
logger.Info("auto metadata retry pass",
|
||||
slog.Int("scanned", out.Scanned),
|
||||
slog.Int("retried", out.Retried),
|
||||
slog.Int("updated", out.Updated),
|
||||
slog.Int("failed", out.Failed),
|
||||
)
|
||||
}
|
||||
|
||||
func runBackfillPass(ctx context.Context, db *store.DB, provider ai.Provider, cfg config.BackfillConfig, logger *slog.Logger) {
|
||||
backfiller := tools.NewBackfillTool(db, provider, nil, logger)
|
||||
_, out, err := backfiller.Handle(ctx, nil, tools.BackfillInput{
|
||||
|
||||
112
internal/app/files.go
Normal file
112
internal/app/files.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.warky.dev/wdevs/amcs/internal/tools"
|
||||
)
|
||||
|
||||
const maxUploadBytes = 50 << 20
|
||||
|
||||
func fileUploadHandler(files *tools.FilesTool) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
w.Header().Set("Allow", http.MethodPost)
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxUploadBytes)
|
||||
|
||||
in, err := parseUploadRequest(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
out, err := files.SaveDecoded(r.Context(), nil, in)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
})
|
||||
}
|
||||
|
||||
func parseUploadRequest(r *http.Request) (tools.SaveFileDecodedInput, error) {
|
||||
contentType := r.Header.Get("Content-Type")
|
||||
mediaType, _, _ := mime.ParseMediaType(contentType)
|
||||
|
||||
if strings.HasPrefix(mediaType, "multipart/form-data") {
|
||||
return parseMultipartUpload(r)
|
||||
}
|
||||
|
||||
return parseRawUpload(r)
|
||||
}
|
||||
|
||||
func parseMultipartUpload(r *http.Request) (tools.SaveFileDecodedInput, error) {
|
||||
if err := r.ParseMultipartForm(maxUploadBytes); err != nil {
|
||||
return tools.SaveFileDecodedInput{}, err
|
||||
}
|
||||
|
||||
file, header, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
return tools.SaveFileDecodedInput{}, errors.New("multipart upload requires a file field named \"file\"")
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
content, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
return tools.SaveFileDecodedInput{}, err
|
||||
}
|
||||
|
||||
return tools.SaveFileDecodedInput{
|
||||
Name: firstNonEmpty(r.FormValue("name"), header.Filename),
|
||||
Content: content,
|
||||
MediaType: firstNonEmpty(r.FormValue("media_type"), header.Header.Get("Content-Type")),
|
||||
Kind: r.FormValue("kind"),
|
||||
ThoughtID: r.FormValue("thought_id"),
|
||||
Project: r.FormValue("project"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseRawUpload(r *http.Request) (tools.SaveFileDecodedInput, error) {
|
||||
content, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return tools.SaveFileDecodedInput{}, err
|
||||
}
|
||||
|
||||
name := firstNonEmpty(
|
||||
r.URL.Query().Get("name"),
|
||||
r.Header.Get("X-File-Name"),
|
||||
)
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return tools.SaveFileDecodedInput{}, errors.New("raw upload requires a file name via query param \"name\" or X-File-Name header")
|
||||
}
|
||||
|
||||
return tools.SaveFileDecodedInput{
|
||||
Name: name,
|
||||
Content: content,
|
||||
MediaType: r.Header.Get("Content-Type"),
|
||||
Kind: firstNonEmpty(r.URL.Query().Get("kind"), r.Header.Get("X-File-Kind")),
|
||||
ThoughtID: firstNonEmpty(r.URL.Query().Get("thought_id"), r.Header.Get("X-Thought-Id")),
|
||||
Project: firstNonEmpty(r.URL.Query().Get("project"), r.Header.Get("X-Project")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
57
internal/app/files_test.go
Normal file
57
internal/app/files_test.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseRawUploadRequiresName(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, "/files", bytes.NewReader([]byte("hello")))
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
|
||||
_, err := parseRawUpload(req)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing name")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMultipartUploadUsesFileMetadata(t *testing.T) {
|
||||
var body bytes.Buffer
|
||||
writer := multipart.NewWriter(&body)
|
||||
|
||||
part, err := writer.CreateFormFile("file", "note.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("create form file: %v", err)
|
||||
}
|
||||
if _, err := part.Write([]byte("hello world")); err != nil {
|
||||
t.Fatalf("write form file: %v", err)
|
||||
}
|
||||
_ = writer.WriteField("project", "amcs")
|
||||
_ = writer.WriteField("kind", "document")
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close writer: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/files", &body)
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
got, err := parseMultipartUpload(req)
|
||||
if err != nil {
|
||||
t.Fatalf("parse multipart upload: %v", err)
|
||||
}
|
||||
if got.Name != "note.txt" {
|
||||
t.Fatalf("name = %q, want note.txt", got.Name)
|
||||
}
|
||||
if string(got.Content) != "hello world" {
|
||||
t.Fatalf("content = %q, want hello world", string(got.Content))
|
||||
}
|
||||
if got.Project != "amcs" {
|
||||
t.Fatalf("project = %q, want amcs", got.Project)
|
||||
}
|
||||
if got.Kind != "document" {
|
||||
t.Fatalf("kind = %q, want document", got.Kind)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user