wdevs/amcs
5
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(backfill): implement backfill tool for generating missing embeddings

Browse Source
This commit is contained in:
Warky
2026-03-26 22:45:28 +02:00
parent 1dde7f233d
commit f4ef0e9163
19 changed files with 575 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
internal/app/app.go
View File

@@ -74,6 +74,25 @@ func Run(ctx context.Context, configPath string) error {
slog.String("provider", provider.Name()),
)
if cfg.Backfill.Enabled && cfg.Backfill.RunOnStartup {
go runBackfillPass(ctx, db, provider, cfg.Backfill, logger)
}
if cfg.Backfill.Enabled && cfg.Backfill.Interval > 0 {
go func() {
ticker := time.NewTicker(cfg.Backfill.Interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
runBackfillPass(ctx, db, provider, cfg.Backfill, logger)
}
}
}()
}
server := &http.Server{
Addr: fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port),
Handler: routes(logger, cfg, db, provider, keyring, oauthRegistry, tokenStore, authCodes, dynClients, activeProjects),
@@ -123,6 +142,7 @@ func routes(logger *slog.Logger, cfg *config.Config, db *store.DB, provider ai.P
Recall: tools.NewRecallTool(db, provider, cfg.Search, activeProjects),
Summarize: tools.NewSummarizeTool(db, provider, cfg.Search, activeProjects),
Links: tools.NewLinksTool(db, provider, cfg.Search),
Backfill: tools.NewBackfillTool(db, provider, activeProjects, logger),
}
mcpHandler := mcpserver.New(cfg.MCP, toolSet)
@@ -136,6 +156,7 @@ func routes(logger *slog.Logger, cfg *config.Config, db *store.DB, provider ai.P
mux.HandleFunc("/oauth/token", oauthTokenHandler(oauthRegistry, tokenStore, authCodes, logger))
}
mux.HandleFunc("/favicon.ico", serveFavicon)
mux.HandleFunc("/images/project.jpg", serveHomeImage)
mux.HandleFunc("/llm", serveLLMInstructions)
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
@@ -155,8 +176,49 @@ func routes(logger *slog.Logger, cfg *config.Config, db *store.DB, provider ai.P
})
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
if r.Method != http.MethodGet && r.Method != http.MethodHead {
w.Header().Set("Allow", "GET, HEAD")
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
const homePage = `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>AMCS</title>
<style>
body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; background: #f5f7fb; color: #172033; }
main { max-width: 860px; margin: 48px auto; background: #fff; border-radius: 12px; box-shadow: 0 10px 28px rgba(23, 32, 51, 0.12); overflow: hidden; }
.content { padding: 28px; }
h1 { margin: 0 0 12px 0; font-size: 2rem; }
p { margin: 0; line-height: 1.5; color: #334155; }
img { display: block; width: 100%; height: auto; }
</style>
</head>
<body>
<main>
<img src="/images/project.jpg" alt="Avelon Memory Crystal project image">
<div class="content">
<h1>Avelon Memory Crystal Server (AMCS)</h1>
<p>AMCS is a memory server that captures, links, and retrieves structured project thoughts for AI assistants using semantic search, summaries, and MCP tools.</p>
</div>
</main>
</body>
</html>`
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("amcs is running"))
if r.Method == http.MethodHead {
return
}
_, _ = w.Write([]byte(homePage))
})
return observability.Chain(
@@ -167,3 +229,39 @@ func routes(logger *slog.Logger, cfg *config.Config, db *store.DB, provider ai.P
observability.Timeout(cfg.Server.WriteTimeout),
)
}
func runBackfillPass(ctx context.Context, db *store.DB, provider ai.Provider, cfg config.BackfillConfig, logger *slog.Logger) {
backfiller := tools.NewBackfillTool(db, provider, nil, logger)
_, out, err := backfiller.Handle(ctx, nil, tools.BackfillInput{
Limit: cfg.MaxPerRun,
IncludeArchived: cfg.IncludeArchived,
})
if err != nil {
logger.Error("auto backfill failed", slog.String("error", err.Error()))
return
}
logger.Info("auto backfill pass",
slog.String("model", out.Model),
slog.Int("scanned", out.Scanned),
slog.Int("embedded", out.Embedded),
slog.Int("failed", out.Failed),
)
}
func serveHomeImage(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet && r.Method != http.MethodHead {
w.Header().Set("Allow", "GET, HEAD")
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
w.Header().Set("Content-Type", "image/jpeg")
w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
w.WriteHeader(http.StatusOK)
if r.Method == http.MethodHead {
return
}
_, _ = w.Write(homeImage)
}