From 767c0fbac69084baeb58b27e7fe3763e894b99bd Mon Sep 17 00:00:00 2001 From: Andre Nogueira Date: Fri, 17 Apr 2026 16:29:20 +0100 Subject: [PATCH] feat: improve indexer resilience for Redis outages, branch renames, and deleted repos The indexer would hang indefinitely when Redis became unreachable, silently mark repos as empty when their default branch was renamed, and waste retries on repos deleted from the code host. Redis outage recovery: - Worker exits after ~5 min of consecutive Redis failures so K8s restarts - CleanupOldJobs now removes ghost entries from running/pending status indexes, type indexes, and processing set - DequeueForShard cleans up ghost job index entries on pop Branch rename detection: - processIndexJob auto-detects the actual default branch when configured branch doesn't exist (prefers main > master > first available) - Updates the database so future jobs use the correct branch Deleted repo detection: - Add PermanentError type to skip retries on unrecoverable failures - Detect 404/not-found git errors during clone/fetch - Mark deleted repos as excluded immediately instead of retrying 3 times - New repo_not_found category in failure metrics Other: - Set fsGroupChangePolicy: OnRootMismatch to speed up pod restarts - golines formatting fixes Signed-off-by: Andre Nogueira Signed-off-by: Andre Nogueira --- .github/workflows/ci.yml | 16 +- .golangci.yml | 3 + Makefile | 32 +- cmd/api/handlers/files.go | 12 +- cmd/api/handlers/repos.go | 24 +- cmd/api/handlers/scip.go | 17 +- cmd/api/handlers/search.go | 6 +- cmd/api/handlers/symbols.go | 19 +- cmd/api/handlers/webhooks.go | 6 +- cmd/api/server/server.go | 4 +- cmd/api/server/server_test.go | 8 +- cmd/cli/cmd/repo.go | 5 +- cmd/cli/cmd/root.go | 6 +- cmd/cli/cmd/search.go | 11 +- cmd/indexer/main.go | 6 +- cmd/mcp/client/client.go | 38 +- cmd/mcp/client/client_test.go | 5 +- cmd/mcp/main.go | 6 +- cmd/mcp/tools/files.go | 28 +- cmd/mcp/tools/repos.go | 30 +- cmd/mcp/tools/scip.go | 21 +- cmd/mcp/tools/search.go | 40 +- deploy/helm/code-search/values.yaml | 1 + internal/cache/invalidator.go | 6 +- internal/indexer/server.go | 17 +- internal/indexer/sharding_test.go | 7 +- internal/indexer/worker.go | 209 +- internal/lock/lock.go | 3 +- internal/queue/queue.go | 138 +- internal/queue/sharded_queue.go | 64 +- internal/replace/service.go | 11 +- internal/repos/service.go | 23 +- internal/scheduler/scheduler.go | 24 +- internal/scip/federated.go | 12 +- internal/scip/indexer.go | 39 +- internal/scip/service.go | 6 +- internal/tracing/middleware_test.go | 21 +- web/.prettierignore | 5 + web/.prettierrc | 7 + web/app/HomeClient.tsx | 171 +- web/app/connections/ConnectionsClient.tsx | 587 ++++-- web/app/connections/page.tsx | 3 +- web/app/jobs/JobsClient.tsx | 478 +++-- web/app/jobs/page.tsx | 3 +- web/app/layout.tsx | 4 +- web/app/page.tsx | 3 +- web/app/replace/ReplaceClient.tsx | 936 ++++++--- web/app/replace/page.tsx | 15 +- web/app/repos/ReposClient.tsx | 932 ++++++--- .../[id]/browse/[[...path]]/BrowseClient.tsx | 1861 +++++++++++------ .../repos/[id]/browse/[[...path]]/page.tsx | 15 +- web/app/repos/page.tsx | 3 +- web/bun.lock | 36 +- web/components/BinaryFileViewer.tsx | 36 +- web/components/BrowseTabBar.tsx | 48 +- web/components/CodeViewer.tsx | 185 +- web/components/ContextManager.tsx | 260 ++- web/components/ContextSidebar.tsx | 45 +- web/components/ContextSwitcher.tsx | 254 ++- web/components/EditorHeader.tsx | 152 +- web/components/FilePane.tsx | 130 +- web/components/FileTree.test.tsx | 270 ++- web/components/FileTree.tsx | 68 +- web/components/FileTreeSidebar.tsx | 23 +- web/components/HoverPopup.tsx | 54 +- web/components/LazyFileTree.tsx | 178 +- web/components/Navigation.tsx | 157 +- web/components/QuickFilePicker.tsx | 88 +- web/components/QuickFilePickerList.tsx | 28 +- web/components/ReferencePanel.tsx | 165 +- web/components/RepoSelector.tsx | 140 +- web/components/Search.tsx | 1039 ++++++--- web/components/SearchDropdown.tsx | 225 +- web/components/SearchInput.tsx | 27 +- web/components/SymbolSidebar.tsx | 57 +- web/components/ThemeToggle.tsx | 15 +- web/hooks/useAuth.tsx | 6 +- web/hooks/useContexts.test.tsx | 364 ++-- web/hooks/useContexts.tsx | 97 +- web/lib/api.ts | 1 - web/lib/code-viewer-utils.ts | 746 +++++-- web/lib/syntax-highlight.tsx | 182 +- web/next.config.js | 8 +- web/package.json | 6 +- web/tsconfig.json | 14 +- 85 files changed, 7387 insertions(+), 3664 deletions(-) create mode 100644 web/.prettierignore create mode 100644 web/.prettierrc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5580f0..9a30335 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,14 +76,16 @@ jobs: - name: Install golangci-lint uses: golangci/golangci-lint-action@v9 with: - args: ./... - - name: Run go vet + install-only: true + - name: Lint + run: make lint-go + - name: Vet run: make vet - - name: Run tests + - name: Test env: CS_DATABASE_URL: postgres://codesearch:codesearch@localhost:5432/codesearch?sslmode=disable CS_REDIS_ADDR: localhost:6379 - run: make test + run: make test-go web-checks: needs: changes @@ -95,9 +97,9 @@ jobs: - name: Install dependencies run: cd web && bun install - name: Lint - run: cd web && bun lint + run: make lint-web - name: Test - run: cd web && bun run test:run + run: make test-web website-checks: needs: changes @@ -109,7 +111,7 @@ jobs: - name: Install dependencies run: cd website && bun install - name: Build - run: cd website && bun run build + run: make build-website docker-build: needs: changes diff --git a/.golangci.yml b/.golangci.yml index ef84eb0..61dfe53 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -141,6 +141,9 @@ linters: exclusions: generated: lax + paths: + - web + - website rules: - path: _test\.go linters: diff --git a/Makefile b/Makefile index eef9c06..99b998a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all build clean test lint help \ +.PHONY: all build build-website clean test test-go test-web lint lint-go lint-web fmt fmt-go fmt-web vet help \ release-api release-indexer release-web release-zoekt release-website \ release-helm release-helm-website release-cli @@ -24,7 +24,7 @@ DEV_ENV := CS_DATABASE_URL=postgres://codesearch:codesearch@localhost:5432/codes all: build -build: ## Build all binaries +build: ## Build all Go binaries $(GOCMD) build -o $(BIN_DIR)/code-search ./$(CMD_DIR)/cli $(GOCMD) build -o $(BIN_DIR)/api-server ./$(CMD_DIR)/api $(GOCMD) build -o $(BIN_DIR)/indexer ./$(CMD_DIR)/indexer @@ -32,6 +32,9 @@ build: ## Build all binaries $(GOCMD) build -o $(BIN_DIR)/zoekt-refresh ./$(CMD_DIR)/zoekt-refresh $(GOCMD) build -o $(BIN_DIR)/mcp-server ./$(CMD_DIR)/mcp +build-website: ## Build project website + cd website && bun run build + # ============================================================================= # Development # ============================================================================= @@ -85,16 +88,35 @@ dev-setup: ## First-time setup for local development # Testing & Linting # ============================================================================= -test: ## Run tests +test: test-go test-web ## Run all tests + +test-go: ## Run Go tests $(GOCMD) test -v ./... -test-cover: ## Run tests with coverage +test-web: ## Run web frontend tests + cd web && bun run test:run + +test-cover: ## Run Go tests with coverage $(GOCMD) test -v -cover -coverprofile=coverage.out ./... $(GOCMD) tool cover -html=coverage.out -o coverage.html -lint: ## Run linters +lint: lint-go lint-web ## Run all linters + +lint-go: ## Run Go linters golangci-lint run ./... +lint-web: ## Run web frontend linter + cd web && bun lint + +fmt: fmt-go fmt-web ## Format all code + +fmt-go: ## Format Go code + gofmt -w . + golines -w . + +fmt-web: ## Format web frontend code (prettier) + cd web && bun run format + vet: ## Run go vet $(GOCMD) vet ./... diff --git a/cmd/api/handlers/files.go b/cmd/api/handlers/files.go index d49ff7e..395cfd4 100644 --- a/cmd/api/handlers/files.go +++ b/cmd/api/handlers/files.go @@ -389,7 +389,11 @@ func (h *Handler) GetBranchesAndTags(w http.ResponseWriter, r *http.Request) { return } - if !h.services.Authorizer.CanAccessRepo(r.Context(), middleware.UserFromContext(r.Context()), repo.Name) { + if !h.services.Authorizer.CanAccessRepo( + r.Context(), + middleware.UserFromContext(r.Context()), + repo.Name, + ) { http.Error(w, "Forbidden", http.StatusForbidden) return } @@ -461,7 +465,11 @@ func (h *Handler) GetFileSymbols(w http.ResponseWriter, r *http.Request) { return } - if !h.services.Authorizer.CanAccessRepo(r.Context(), middleware.UserFromContext(r.Context()), repo.Name) { + if !h.services.Authorizer.CanAccessRepo( + r.Context(), + middleware.UserFromContext(r.Context()), + repo.Name, + ) { http.Error(w, "Forbidden", http.StatusForbidden) return } diff --git a/cmd/api/handlers/repos.go b/cmd/api/handlers/repos.go index 0ad2cee..bad01f5 100644 --- a/cmd/api/handlers/repos.go +++ b/cmd/api/handlers/repos.go @@ -227,7 +227,11 @@ func (h *Handler) GetRepoByID(w http.ResponseWriter, r *http.Request) { return } - if !h.services.Authorizer.CanAccessRepo(r.Context(), middleware.UserFromContext(r.Context()), repo.Name) { + if !h.services.Authorizer.CanAccessRepo( + r.Context(), + middleware.UserFromContext(r.Context()), + repo.Name, + ) { http.Error(w, "Forbidden", http.StatusForbidden) return } @@ -463,7 +467,11 @@ func (h *Handler) IncludeRepoByID(w http.ResponseWriter, r *http.Request) { acquired, err := h.services.Queue.TryAcquireIndexJob(r.Context(), repo.ID) if err != nil { // Log but don't fail - queue check is best-effort - log.L.Debug("Failed to acquire index job slot", zap.Int64("repo_id", repo.ID), zap.Error(err)) + log.L.Debug( + "Failed to acquire index job slot", + zap.Int64("repo_id", repo.ID), + zap.Error(err), + ) } var jobID string @@ -548,7 +556,11 @@ func (h *Handler) RestoreRepoByID(w http.ResponseWriter, r *http.Request) { // Queue an index job immediately so it doesn't wait for scheduler acquired, err := h.services.Queue.TryAcquireIndexJob(r.Context(), repo.ID) if err != nil { - log.L.Debug("Failed to acquire index job slot", zap.Int64("repo_id", repo.ID), zap.Error(err)) + log.L.Debug( + "Failed to acquire index job slot", + zap.Int64("repo_id", repo.ID), + zap.Error(err), + ) } var jobID string @@ -630,7 +642,11 @@ func (h *Handler) SyncRepoByID(w http.ResponseWriter, r *http.Request) { if repo.IndexStatus == "pending" { active, err := h.services.Queue.HasPendingIndexJob(r.Context(), repo.ID) if err != nil { - log.L.Debug("Failed to check active index job", zap.Int64("repo_id", repo.ID), zap.Error(err)) + log.L.Debug( + "Failed to check active index job", + zap.Int64("repo_id", repo.ID), + zap.Error(err), + ) active = true } diff --git a/cmd/api/handlers/scip.go b/cmd/api/handlers/scip.go index dab142a..298ce9a 100644 --- a/cmd/api/handlers/scip.go +++ b/cmd/api/handlers/scip.go @@ -30,7 +30,10 @@ func NewSCIPHandler(services *Services, scipSvc *scip.Service) *SCIPHandler { } // getBlob fetches file content using federated access when available. -func (h *SCIPHandler) getBlob(ctx context.Context, repoName, path, ref string) (*files.BlobResponse, error) { +func (h *SCIPHandler) getBlob( + ctx context.Context, + repoName, path, ref string, +) (*files.BlobResponse, error) { if h.services.FederatedFiles != nil { return h.services.FederatedFiles.GetBlob(ctx, repoName, path, ref) } @@ -116,7 +119,11 @@ func (h *SCIPHandler) checkRepoAccess(w http.ResponseWriter, r *http.Request, re return false } - if !h.services.Authorizer.CanAccessRepo(r.Context(), middleware.UserFromContext(r.Context()), repo.Name) { + if !h.services.Authorizer.CanAccessRepo( + r.Context(), + middleware.UserFromContext(r.Context()), + repo.Name, + ) { http.Error(w, "Forbidden", http.StatusForbidden) return false } @@ -352,7 +359,11 @@ func (h *SCIPHandler) IndexRepository(w http.ResponseWriter, r *http.Request) { return } - if !h.services.Authorizer.CanAccessRepo(r.Context(), middleware.UserFromContext(r.Context()), repo.Name) { + if !h.services.Authorizer.CanAccessRepo( + r.Context(), + middleware.UserFromContext(r.Context()), + repo.Name, + ) { http.Error(w, "Forbidden", http.StatusForbidden) return } diff --git a/cmd/api/handlers/search.go b/cmd/api/handlers/search.go index 627ed08..9aa2071 100644 --- a/cmd/api/handlers/search.go +++ b/cmd/api/handlers/search.go @@ -521,7 +521,11 @@ func (h *Handler) SearchSuggestions(w http.ResponseWriter, r *http.Request) { Example: "content:FOO", }, {Keyword: "branch:", Description: "Filter by branch/tag", Example: "branch:main"}, - {Keyword: "type:", Description: "Result type: filematch, filename, or repo", Example: "type:filename main"}, + { + Keyword: "type:", + Description: "Result type: filematch, filename, or repo", + Example: "type:filename main", + }, {Keyword: "regex:", Description: "Treat pattern as regex", Example: "regex:func\\s+main"}, {Keyword: "-repo:", Description: "Exclude repository", Example: "-repo:test"}, {Keyword: "-file:", Description: "Exclude file pattern", Example: "-file:*_test.go"}, diff --git a/cmd/api/handlers/symbols.go b/cmd/api/handlers/symbols.go index 381da46..5bcb55c 100644 --- a/cmd/api/handlers/symbols.go +++ b/cmd/api/handlers/symbols.go @@ -58,7 +58,11 @@ func (h *Handler) FindSymbols(w http.ResponseWriter, r *http.Request) { // RBAC: filter requested repos to only those the user can access if len(req.Repos) > 0 { - req.Repos = h.services.Authorizer.FilterRepos(r.Context(), middleware.UserFromContext(r.Context()), req.Repos) + req.Repos = h.services.Authorizer.FilterRepos( + r.Context(), + middleware.UserFromContext(r.Context()), + req.Repos, + ) } // Build Zoekt query with sym: filter for symbol search @@ -160,7 +164,11 @@ func (h *Handler) FindRefs(w http.ResponseWriter, r *http.Request) { // RBAC: filter requested repos to only those the user can access if len(req.Repos) > 0 { - req.Repos = h.services.Authorizer.FilterRepos(r.Context(), middleware.UserFromContext(r.Context()), req.Repos) + req.Repos = h.services.Authorizer.FilterRepos( + r.Context(), + middleware.UserFromContext(r.Context()), + req.Repos, + ) } // Build Zoekt query - search for the symbol as a literal string @@ -558,7 +566,12 @@ func (h *Handler) TestConnection(w http.ResponseWriter, r *http.Request) { // The actual fetching is done asynchronously by the indexer worker. func (h *Handler) SyncConnection(w http.ResponseWriter, r *http.Request) { if h.connectionsReadOnly { - http.Error(w, "Connections are read-only. Manage connections via config file.", http.StatusForbidden) + http.Error( + w, + "Connections are read-only. Manage connections via config file.", + http.StatusForbidden, + ) + return } diff --git a/cmd/api/handlers/webhooks.go b/cmd/api/handlers/webhooks.go index a1393dd..d4d36e1 100644 --- a/cmd/api/handlers/webhooks.go +++ b/cmd/api/handlers/webhooks.go @@ -70,7 +70,11 @@ func (h *Handler) HandleWebhook(w http.ResponseWriter, r *http.Request) { } if repoName == "" { - http.Error(w, "Could not extract repository name from webhook payload", http.StatusBadRequest) + http.Error( + w, + "Could not extract repository name from webhook payload", + http.StatusBadRequest, + ) return } diff --git a/cmd/api/server/server.go b/cmd/api/server/server.go index 3b37b56..344c0aa 100644 --- a/cmd/api/server/server.go +++ b/cmd/api/server/server.go @@ -369,7 +369,9 @@ func (b *Builder) WithAuditLogger(al audit.AuditLogger) *Builder { // WithSearchResultFilter sets a function to filter search results based on // user access. This is a convenience that wraps authorizer-level filtering // into the search pipeline. -func (b *Builder) WithSearchResultFilter(fn func(ctx context.Context, results []search.SearchResult) []search.SearchResult) *Builder { +func (b *Builder) WithSearchResultFilter( + fn func(ctx context.Context, results []search.SearchResult) []search.SearchResult, +) *Builder { b.services.SearchResultFilter = fn return b } diff --git a/cmd/api/server/server_test.go b/cmd/api/server/server_test.go index 20ce105..8598c5f 100644 --- a/cmd/api/server/server_test.go +++ b/cmd/api/server/server_test.go @@ -68,7 +68,13 @@ func TestSecurityHeaders_PresentOnAllMethods(t *testing.T) { }) handler := securityHeaders(inner) - methods := []string{http.MethodGet, http.MethodPost, http.MethodPut, http.MethodDelete, http.MethodOptions} + methods := []string{ + http.MethodGet, + http.MethodPost, + http.MethodPut, + http.MethodDelete, + http.MethodOptions, + } for _, method := range methods { req := httptest.NewRequest(method, "/", nil) rr := httptest.NewRecorder() diff --git a/cmd/cli/cmd/repo.go b/cmd/cli/cmd/repo.go index 1c177dc..6d89194 100644 --- a/cmd/cli/cmd/repo.go +++ b/cmd/cli/cmd/repo.go @@ -283,7 +283,10 @@ func runRepoRemove(cmd *cobra.Command, args []string) error { force, _ := cmd.Flags().GetBool("force") if !force { - fmt.Printf("Remove repository ID '%d'? This will soft-delete the repository (can be restored later). [y/N]: ", repoID) + fmt.Printf( + "Remove repository ID '%d'? This will soft-delete the repository (can be restored later). [y/N]: ", + repoID, + ) reader := bufio.NewReader(os.Stdin) response, _ := reader.ReadString('\n') diff --git a/cmd/cli/cmd/root.go b/cmd/cli/cmd/root.go index e5e1a18..f085d7a 100644 --- a/cmd/cli/cmd/root.go +++ b/cmd/cli/cmd/root.go @@ -52,8 +52,10 @@ func init() { rootCmd.PersistentFlags(). StringVar(&apiURL, "api-url", "http://localhost:8080", "API server URL") rootCmd.PersistentFlags().String("output", "text", "Output format: text, json, table") - rootCmd.PersistentFlags().String("token", "", "Code host token for replace operations (GitHub/GitLab PAT)") - rootCmd.PersistentFlags().String("auth-token", "", "Authentication token (JWT or API token, env: CODE_SEARCH_AUTH_TOKEN)") + rootCmd.PersistentFlags(). + String("token", "", "Code host token for replace operations (GitHub/GitLab PAT)") + rootCmd.PersistentFlags(). + String("auth-token", "", "Authentication token (JWT or API token, env: CODE_SEARCH_AUTH_TOKEN)") rootCmd.PersistentFlags(). StringVar(&iapClientID, "iap-client-id", "", "IAP OAuth client ID (for servers behind Google IAP)") diff --git a/cmd/cli/cmd/search.go b/cmd/cli/cmd/search.go index c6eb730..f18f761 100644 --- a/cmd/cli/cmd/search.go +++ b/cmd/cli/cmd/search.go @@ -111,8 +111,12 @@ func outputSearchText(resp *client.SearchResponse) error { fileKey := result.Repo + "/" + result.File last := len(groups) - 1 - if last >= 0 && groups[last].result.Repo+"/"+groups[last].result.File == fileKey && groups[last].result.Line == result.Line { - groups[last].ranges = append(groups[last].ranges, [2]int{result.MatchStart, result.MatchEnd}) + if last >= 0 && groups[last].result.Repo+"/"+groups[last].result.File == fileKey && + groups[last].result.Line == result.Line { + groups[last].ranges = append( + groups[last].ranges, + [2]int{result.MatchStart, result.MatchEnd}, + ) } else { groups = append(groups, lineGroup{ result: result, @@ -350,7 +354,8 @@ func runStreamSearch(c *client.Client, req *client.SearchRequest, output string) fileKey := result.Repo + "/" + result.File // Check if this is a same-line match we can merge - if pendingResult != nil && pendingResult.Repo+"/"+pendingResult.File == fileKey && pendingResult.Line == result.Line { + if pendingResult != nil && pendingResult.Repo+"/"+pendingResult.File == fileKey && + pendingResult.Line == result.Line { pendingRanges = append(pendingRanges, [2]int{result.MatchStart, result.MatchEnd}) return nil } diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go index 545e9df..c78c681 100644 --- a/cmd/indexer/main.go +++ b/cmd/indexer/main.go @@ -153,7 +153,11 @@ func main() { } } - svc, scipErr := scip.NewServiceWithConfig(scipCacheDir, indexerCfg, log.L.With(zap.String("component", "scip"))) + svc, scipErr := scip.NewServiceWithConfig( + scipCacheDir, + indexerCfg, + log.L.With(zap.String("component", "scip")), + ) if scipErr != nil { log.Warn("Failed to initialize SCIP service, auto-indexing disabled", log.Err(scipErr)) } else { diff --git a/cmd/mcp/client/client.go b/cmd/mcp/client/client.go index ebdb4d8..b15e336 100644 --- a/cmd/mcp/client/client.go +++ b/cmd/mcp/client/client.go @@ -209,7 +209,10 @@ type ListReposParams struct { } // ListRepos lists indexed repositories. -func (c *Client) ListRepos(ctx context.Context, params ListReposParams) (*ListReposResponse, error) { +func (c *Client) ListRepos( + ctx context.Context, + params ListReposParams, +) (*ListReposResponse, error) { q := url.Values{} if params.Search != "" { q.Set("search", params.Search) @@ -262,7 +265,11 @@ type TreeResponse struct { } // GetFileTree returns directory contents for a repository. -func (c *Client) GetFileTree(ctx context.Context, repoID int64, path, ref string) (*TreeResponse, error) { +func (c *Client) GetFileTree( + ctx context.Context, + repoID int64, + path, ref string, +) (*TreeResponse, error) { q := url.Values{} if path != "" { q.Set("path", path) @@ -301,7 +308,11 @@ type BlobResponse struct { } // GetFileContent returns the content of a file. -func (c *Client) GetFileContent(ctx context.Context, repoID int64, path, ref string) (*BlobResponse, error) { +func (c *Client) GetFileContent( + ctx context.Context, + repoID int64, + path, ref string, +) (*BlobResponse, error) { q := url.Values{} q.Set("path", path) @@ -364,7 +375,12 @@ type SearchSymbolsResponse struct { } // SearchSymbols searches for symbols in a repository. -func (c *Client) SearchSymbols(ctx context.Context, repoID int64, query string, limit int) (*SearchSymbolsResponse, error) { +func (c *Client) SearchSymbols( + ctx context.Context, + repoID int64, + query string, + limit int, +) (*SearchSymbolsResponse, error) { body := map[string]any{ "query": query, } @@ -415,7 +431,12 @@ type GoToDefinitionResponse struct { } // GoToDefinition finds the definition of a symbol. -func (c *Client) GoToDefinition(ctx context.Context, repoID int64, file string, line, column int) (*GoToDefinitionResponse, error) { +func (c *Client) GoToDefinition( + ctx context.Context, + repoID int64, + file string, + line, column int, +) (*GoToDefinitionResponse, error) { body := map[string]any{ "filePath": file, "line": line, @@ -447,7 +468,12 @@ type FindReferencesResponse struct { } // FindReferences finds all references to a symbol. -func (c *Client) FindReferences(ctx context.Context, repoID int64, file string, line, column, limit int) (*FindReferencesResponse, error) { +func (c *Client) FindReferences( + ctx context.Context, + repoID int64, + file string, + line, column, limit int, +) (*FindReferencesResponse, error) { body := map[string]any{ "filePath": file, "line": line, diff --git a/cmd/mcp/client/client_test.go b/cmd/mcp/client/client_test.go index a5187f1..56c82e7 100644 --- a/cmd/mcp/client/client_test.go +++ b/cmd/mcp/client/client_test.go @@ -95,6 +95,9 @@ func TestAuthTokenWhitespaceOnly(t *testing.T) { _, _ = c.get(context.Background(), "/test") if gotAuth != "" { - t.Errorf("Authorization header = %q, want empty (whitespace-only token should be treated as empty)", gotAuth) + t.Errorf( + "Authorization header = %q, want empty (whitespace-only token should be treated as empty)", + gotAuth, + ) } } diff --git a/cmd/mcp/main.go b/cmd/mcp/main.go index 2065fac..b6e6af1 100644 --- a/cmd/mcp/main.go +++ b/cmd/mcp/main.go @@ -17,7 +17,11 @@ func main() { log.SetOutput(os.Stderr) apiURL := flag.String("api-url", "http://localhost:8080", "Code Search API URL") - authToken := flag.String("auth-token", "", "Authentication token for the Code Search API (env: CODE_SEARCH_AUTH_TOKEN)") + authToken := flag.String( + "auth-token", + "", + "Authentication token for the Code Search API (env: CODE_SEARCH_AUTH_TOKEN)", + ) transport := flag.String("transport", "stdio", "Transport type: stdio or http") httpAddr := flag.String("http-addr", ":9090", "HTTP listen address (for http transport)") diff --git a/cmd/mcp/tools/files.go b/cmd/mcp/tools/files.go index 353d5a0..8fefad9 100644 --- a/cmd/mcp/tools/files.go +++ b/cmd/mcp/tools/files.go @@ -13,8 +13,11 @@ import ( // GetFileTreeTool returns the tool definition for get_file_tree. func GetFileTreeTool() mcp.Tool { - return mcp.NewTool("get_file_tree", - mcp.WithDescription("Browse directory contents in a repository. Returns files and subdirectories at the specified path."), + return mcp.NewTool( + "get_file_tree", + mcp.WithDescription( + "Browse directory contents in a repository. Returns files and subdirectories at the specified path.", + ), mcp.WithNumber("repo_id", mcp.Required(), mcp.Description("Repository ID (use list_repos to find IDs)"), @@ -22,8 +25,11 @@ func GetFileTreeTool() mcp.Tool { mcp.WithString("path", mcp.Description("Directory path to list (default: root '/')"), ), - mcp.WithString("ref", - mcp.Description("Git ref (branch, tag, or commit SHA) to browse (default: repository's default branch)"), + mcp.WithString( + "ref", + mcp.Description( + "Git ref (branch, tag, or commit SHA) to browse (default: repository's default branch)", + ), ), ) } @@ -50,8 +56,11 @@ func HandleGetFileTree(c *client.Client) server.ToolHandlerFunc { // GetFileContentTool returns the tool definition for get_file_content. func GetFileContentTool() mcp.Tool { - return mcp.NewTool("get_file_content", - mcp.WithDescription("Read the content of a file in a repository. Returns the file content with syntax highlighting language detection. Binary files return a message instead of content."), + return mcp.NewTool( + "get_file_content", + mcp.WithDescription( + "Read the content of a file in a repository. Returns the file content with syntax highlighting language detection. Binary files return a message instead of content.", + ), mcp.WithNumber("repo_id", mcp.Required(), mcp.Description("Repository ID (use list_repos to find IDs)"), @@ -60,8 +69,11 @@ func GetFileContentTool() mcp.Tool { mcp.Required(), mcp.Description("File path within the repository"), ), - mcp.WithString("ref", - mcp.Description("Git ref (branch, tag, or commit SHA) to read from (default: repository's default branch)"), + mcp.WithString( + "ref", + mcp.Description( + "Git ref (branch, tag, or commit SHA) to read from (default: repository's default branch)", + ), ), ) } diff --git a/cmd/mcp/tools/repos.go b/cmd/mcp/tools/repos.go index f3dd36e..0d8a40e 100644 --- a/cmd/mcp/tools/repos.go +++ b/cmd/mcp/tools/repos.go @@ -13,8 +13,11 @@ import ( // ListReposTool returns the tool definition for list_repos. func ListReposTool() mcp.Tool { - return mcp.NewTool("list_repos", - mcp.WithDescription("List indexed repositories. Use this to discover available repositories and their IDs, which are needed for other tools like get_file_tree, get_file_content, and SCIP tools."), + return mcp.NewTool( + "list_repos", + mcp.WithDescription( + "List indexed repositories. Use this to discover available repositories and their IDs, which are needed for other tools like get_file_tree, get_file_content, and SCIP tools.", + ), mcp.WithString("search", mcp.Description("Filter repositories by name (substring match)"), ), @@ -51,8 +54,11 @@ func HandleListRepos(c *client.Client) server.ToolHandlerFunc { // GetRepoBranchesTool returns the tool definition for get_repo_branches. func GetRepoBranchesTool() mcp.Tool { - return mcp.NewTool("get_repo_branches", - mcp.WithDescription("List branches and tags for a repository. Returns the default branch, all branches, and all tags."), + return mcp.NewTool( + "get_repo_branches", + mcp.WithDescription( + "List branches and tags for a repository. Returns the default branch, all branches, and all tags.", + ), mcp.WithNumber("repo_id", mcp.Required(), mcp.Description("Repository ID (use list_repos to find IDs)"), @@ -84,7 +90,15 @@ func formatListRepos(resp *client.ListReposResponse) string { var sb strings.Builder fmt.Fprintf(&sb, "Repositories (showing %d of %d):\n\n", len(resp.Repos), resp.TotalCount) - fmt.Fprintf(&sb, "%-6s %-50s %-10s %-10s %s\n", "ID", "Name", "Status", "Branch", "Last Indexed") + fmt.Fprintf( + &sb, + "%-6s %-50s %-10s %-10s %s\n", + "ID", + "Name", + "Status", + "Branch", + "Last Indexed", + ) sb.WriteString(strings.Repeat("-", 110) + "\n") for _, r := range resp.Repos { @@ -103,7 +117,11 @@ func formatListRepos(resp *client.ListReposResponse) string { } if resp.HasMore { - fmt.Fprintf(&sb, "\n[More results available. Use offset=%d to see next page.]", resp.Offset+len(resp.Repos)) + fmt.Fprintf( + &sb, + "\n[More results available. Use offset=%d to see next page.]", + resp.Offset+len(resp.Repos), + ) } return sb.String() diff --git a/cmd/mcp/tools/scip.go b/cmd/mcp/tools/scip.go index d788cda..072f9eb 100644 --- a/cmd/mcp/tools/scip.go +++ b/cmd/mcp/tools/scip.go @@ -14,8 +14,11 @@ import ( // SearchSymbolsTool returns the tool definition for search_symbols. func SearchSymbolsTool() mcp.Tool { - return mcp.NewTool("search_symbols", - mcp.WithDescription("Search for symbol definitions (functions, classes, methods, variables) in a repository using SCIP code intelligence. Requires SCIP indexing to be enabled on the server."), + return mcp.NewTool( + "search_symbols", + mcp.WithDescription( + "Search for symbol definitions (functions, classes, methods, variables) in a repository using SCIP code intelligence. Requires SCIP indexing to be enabled on the server.", + ), mcp.WithNumber("repo_id", mcp.Required(), mcp.Description("Repository ID (use list_repos to find IDs)"), @@ -56,8 +59,11 @@ func HandleSearchSymbols(c *client.Client) server.ToolHandlerFunc { // GoToDefinitionTool returns the tool definition for go_to_definition. func GoToDefinitionTool() mcp.Tool { - return mcp.NewTool("go_to_definition", - mcp.WithDescription("Jump to the definition of a symbol at a specific location in a file. Uses SCIP code intelligence for precise navigation. Line is 1-indexed, column is 0-indexed."), + return mcp.NewTool( + "go_to_definition", + mcp.WithDescription( + "Jump to the definition of a symbol at a specific location in a file. Uses SCIP code intelligence for precise navigation. Line is 1-indexed, column is 0-indexed.", + ), mcp.WithNumber("repo_id", mcp.Required(), mcp.Description("Repository ID (use list_repos to find IDs)"), @@ -111,8 +117,11 @@ func HandleGoToDefinition(c *client.Client) server.ToolHandlerFunc { // FindReferencesTool returns the tool definition for find_references. func FindReferencesTool() mcp.Tool { - return mcp.NewTool("find_references", - mcp.WithDescription("Find all usages/references of a symbol at a specific location in a file. Uses SCIP code intelligence. Line is 1-indexed, column is 0-indexed."), + return mcp.NewTool( + "find_references", + mcp.WithDescription( + "Find all usages/references of a symbol at a specific location in a file. Uses SCIP code intelligence. Line is 1-indexed, column is 0-indexed.", + ), mcp.WithNumber("repo_id", mcp.Required(), mcp.Description("Repository ID (use list_repos to find IDs)"), diff --git a/cmd/mcp/tools/search.go b/cmd/mcp/tools/search.go index 71fa9fc..9b2bc19 100644 --- a/cmd/mcp/tools/search.go +++ b/cmd/mcp/tools/search.go @@ -14,20 +14,35 @@ import ( // SearchCodeTool returns the tool definition for search_code. func SearchCodeTool() mcp.Tool { - return mcp.NewTool("search_code", - mcp.WithDescription("Search for code patterns across all indexed repositories. Supports text and regex search with filtering by repo, language, and file pattern."), - mcp.WithString("query", + return mcp.NewTool( + "search_code", + mcp.WithDescription( + "Search for code patterns across all indexed repositories. Supports text and regex search with filtering by repo, language, and file pattern.", + ), + mcp.WithString( + "query", mcp.Required(), - mcp.Description("Search query (text or regex pattern). Supports Zoekt query syntax including operators like repo:, file:, lang:, case:yes, sym:, branch:"), + mcp.Description( + "Search query (text or regex pattern). Supports Zoekt query syntax including operators like repo:, file:, lang:, case:yes, sym:, branch:", + ), ), - mcp.WithString("repos", - mcp.Description("Comma-separated list of repository name patterns to search in (regex). Example: 'org/repo1,org/repo2'"), + mcp.WithString( + "repos", + mcp.Description( + "Comma-separated list of repository name patterns to search in (regex). Example: 'org/repo1,org/repo2'", + ), ), - mcp.WithString("languages", - mcp.Description("Comma-separated list of languages to filter by. Example: 'go,typescript'"), + mcp.WithString( + "languages", + mcp.Description( + "Comma-separated list of languages to filter by. Example: 'go,typescript'", + ), ), - mcp.WithString("file_patterns", - mcp.Description("Comma-separated list of file path patterns to filter by. Example: '*.go,*.ts'"), + mcp.WithString( + "file_patterns", + mcp.Description( + "Comma-separated list of file path patterns to filter by. Example: '*.go,*.ts'", + ), ), mcp.WithBoolean("is_regex", mcp.Description("Treat query as a regular expression"), @@ -170,7 +185,10 @@ func formatAPIError(operation string, err error) string { } if strings.Contains(err.Error(), "connect to Code Search API") { - return fmt.Sprintf("Failed to connect to Code Search API. Is the API server running? Error: %v", err) + return fmt.Sprintf( + "Failed to connect to Code Search API. Is the API server running? Error: %v", + err, + ) } return fmt.Sprintf("%s failed: %v", operation, err) diff --git a/deploy/helm/code-search/values.yaml b/deploy/helm/code-search/values.yaml index a627ce0..e43556b 100644 --- a/deploy/helm/code-search/values.yaml +++ b/deploy/helm/code-search/values.yaml @@ -288,6 +288,7 @@ serviceAccount: # Pod Security podSecurityContext: fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch seccompProfile: type: RuntimeDefault diff --git a/internal/cache/invalidator.go b/internal/cache/invalidator.go index d6a9033..6986a9f 100644 --- a/internal/cache/invalidator.go +++ b/internal/cache/invalidator.go @@ -67,7 +67,11 @@ func (inv *Invalidator) Start(ctx context.Context) { repoID, err := strconv.ParseInt(msg.Payload, 10, 64) if err != nil { - inv.logger.Debug("Invalid reindex event payload", zap.String("payload", msg.Payload)) + inv.logger.Debug( + "Invalid reindex event payload", + zap.String("payload", msg.Payload), + ) + continue } diff --git a/internal/indexer/server.go b/internal/indexer/server.go index 8800913..5124187 100644 --- a/internal/indexer/server.go +++ b/internal/indexer/server.go @@ -400,7 +400,13 @@ func (s *Server) handleSCIPDefinition(w http.ResponseWriter, r *http.Request) { return } - result, err := s.scipService.GoToDefinition(r.Context(), repoID, req.FilePath, req.Line, req.Column) + result, err := s.scipService.GoToDefinition( + r.Context(), + repoID, + req.FilePath, + req.Line, + req.Column, + ) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return @@ -437,7 +443,14 @@ func (s *Server) handleSCIPReferences(w http.ResponseWriter, r *http.Request) { req.Limit = 100 } - result, err := s.scipService.FindReferences(r.Context(), repoID, req.FilePath, req.Line, req.Column, req.Limit) + result, err := s.scipService.FindReferences( + r.Context(), + repoID, + req.FilePath, + req.Line, + req.Column, + req.Limit, + ) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return diff --git a/internal/indexer/sharding_test.go b/internal/indexer/sharding_test.go index 91e44f1..0eb7d50 100644 --- a/internal/indexer/sharding_test.go +++ b/internal/indexer/sharding_test.go @@ -137,7 +137,12 @@ func TestExtractOrdinal(t *testing.T) { t.Run(tt.podName, func(t *testing.T) { result := sharding.ExtractOrdinal(tt.podName) if result != tt.expected { - t.Errorf("sharding.ExtractOrdinal(%q) = %v, want %v", tt.podName, result, tt.expected) + t.Errorf( + "sharding.ExtractOrdinal(%q) = %v, want %v", + tt.podName, + result, + tt.expected, + ) } }) } diff --git a/internal/indexer/worker.go b/internal/indexer/worker.go index bfce18d..ffeafbb 100644 --- a/internal/indexer/worker.go +++ b/internal/indexer/worker.go @@ -266,7 +266,12 @@ func (w *Worker) isSCIPLanguageEnabled(language string) bool { // runSCIPIndexing runs SCIP code intelligence indexing after successful Zoekt indexing. // It detects all languages in the repo and indexes each enabled/available one. // This is non-fatal: errors are logged as warnings but never fail the index job. -func (w *Worker) runSCIPIndexing(ctx context.Context, repoID int64, repoPath, repoName string, conn *repos.Connection) { +func (w *Worker) runSCIPIndexing( + ctx context.Context, + repoID int64, + repoPath, repoName string, + conn *repos.Connection, +) { if w.scipService == nil || !w.cfg.SCIP.Enabled || !w.cfg.SCIP.AutoIndex { return } @@ -394,6 +399,12 @@ func (w *Worker) updateIndexStatusBestEffort( } } +// redisHealthMaxFailures is the number of consecutive Redis failures before +// the worker exits to allow K8s to restart the pod. +// With a 1-second sleep between retries and 5-second dequeue timeout, +// this gives roughly 5 minutes of Redis downtime before exiting. +const redisHealthMaxFailures = 50 + // Run starts the worker loop. // It supports graceful shutdown: when ctx is canceled, it will finish // processing the current job before returning (up to shutdownTimeout). @@ -407,6 +418,8 @@ func (w *Worker) Run(ctx context.Context) error { // Uses Redis lock to ensure only one indexer runs recovery, regardless of shard index. go w.runRecoveryWithLeaderElection(ctx) + consecutiveFailures := 0 + for { select { case <-ctx.Done(): @@ -422,12 +435,31 @@ func (w *Worker) Run(ctx context.Context) error { return nil } - w.logger.Error("Failed to dequeue job", zap.Error(err)) + consecutiveFailures++ + w.logger.Error("Failed to dequeue job", + zap.Error(err), + zap.Int("consecutive_failures", consecutiveFailures), + ) + + if consecutiveFailures >= redisHealthMaxFailures { + w.logger.Error("Redis unreachable for too long, exiting to allow restart", + zap.Int("consecutive_failures", consecutiveFailures), + ) + + return fmt.Errorf( + "redis health check failed: %d consecutive dequeue failures", + consecutiveFailures, + ) + } + time.Sleep(1 * time.Second) continue } + // Reset failure counter on successful dequeue (including timeout with no job) + consecutiveFailures = 0 + if job == nil { // No job available, continue waiting continue @@ -577,7 +609,8 @@ func (w *Worker) runRecoveryWithLeaderElection(ctx context.Context) { // Release leadership on shutdown if isLeader { script := `if redis.call("GET", KEYS[1]) == ARGV[1] then return redis.call("DEL", KEYS[1]) else return 0 end` - _, _ = w.redisClient.Eval(ctx, script, []string{recoveryLeaderKey}, w.workerID).Result() + _, _ = w.redisClient.Eval(ctx, script, []string{recoveryLeaderKey}, w.workerID). + Result() } return @@ -645,7 +678,9 @@ func (w *Worker) extendLock(ctx context.Context, repoLock *lock.DistributedLock, // This prevents CleanupStaleIndexing from resetting long-running indexing jobs. // Call this in a goroutine for long-running indexing operations. func (w *Worker) sendRepoHeartbeats(ctx context.Context, repoID int64, repoName string) { - ticker := time.NewTicker(10 * time.Minute) // Touch every 10 minutes (stale threshold is 1+ hours) + ticker := time.NewTicker( + 10 * time.Minute, + ) // Touch every 10 minutes (stale threshold is 1+ hours) defer ticker.Stop() for { @@ -780,6 +815,10 @@ func (w *Worker) processIndexJob(ctx context.Context, job *queue.Job) error { } // Clone the repository if _, err := w.CloneRepository(ctx, payload.CloneURL, payload.RepoName, conn); err != nil { + if isRepoNotFoundError(err) { + return w.handleRepoNotFound(ctx, payload, logger, start, err) + } + w.updateIndexStatusBestEffort(ctx, payload.RepositoryID, "failed") metrics.RecordJob("index", time.Since(start), false) @@ -796,11 +835,22 @@ func (w *Worker) processIndexJob(ctx context.Context, job *queue.Job) error { // Fetch updates err = w.FetchRepository(ctx, repoPath, conn) if err != nil { + // If repo is gone from remote, no point re-cloning + if isRepoNotFoundError(err) { + _ = os.RemoveAll(repoPath) + + return w.handleRepoNotFound(ctx, payload, logger, start, err) + } + w.logger.Warn("Failed to fetch, will try to re-clone", zap.Error(err)) // Try removing and re-cloning _ = os.RemoveAll(repoPath) if _, err := w.CloneRepository(ctx, payload.CloneURL, payload.RepoName, conn); err != nil { + if isRepoNotFoundError(err) { + return w.handleRepoNotFound(ctx, payload, logger, start, err) + } + w.updateIndexStatusBestEffort(ctx, payload.RepositoryID, "failed") metrics.RecordJob("index", time.Since(start), false) @@ -907,8 +957,11 @@ func (w *Worker) processIndexJob(ctx context.Context, job *queue.Job) error { // Check repo size if configured if w.cfg.Indexer.MaxRepoSizeMB > 0 && repoSizeMB > w.cfg.Indexer.MaxRepoSizeMB { - msg := fmt.Sprintf("repository size (%d MB) exceeds max_repo_size_mb limit (%d MB), skipping indexing", - repoSizeMB, w.cfg.Indexer.MaxRepoSizeMB) + msg := fmt.Sprintf( + "repository size (%d MB) exceeds max_repo_size_mb limit (%d MB), skipping indexing", + repoSizeMB, + w.cfg.Indexer.MaxRepoSizeMB, + ) logger.Warn(msg, zap.Int64("repo_size_mb", repoSizeMB), @@ -947,14 +1000,31 @@ func (w *Worker) processIndexJob(ctx context.Context, job *queue.Job) error { } if len(validBranches) == 0 { - logger.Warn("No valid branches found, repository may be empty", - zap.Strings("requested_branches", branches), - ) + // Before marking as empty, check if the repo has any branches at all. + // The configured branch may have been renamed (e.g. master → main). + detectedBranch := w.detectDefaultBranch(ctx, repoPath) + if detectedBranch != "" { + logger.Info("Detected branch rename, using actual default branch", + zap.String("old_branch", payload.Branch), + zap.String("detected_branch", detectedBranch), + zap.Strings("requested_branches", branches), + ) - w.updateIndexStatusBestEffort(ctx, payload.RepositoryID, "empty") - metrics.RecordJob("index", time.Since(start), true) + if err := w.reposService.UpdateDefaultBranch(ctx, payload.RepositoryID, detectedBranch); err != nil { + logger.Warn("Failed to update default branch in database", zap.Error(err)) + } - return nil + validBranches = []string{detectedBranch} + } else { + logger.Warn("No valid branches found, repository is empty", + zap.Strings("requested_branches", branches), + ) + + w.updateIndexStatusBestEffort(ctx, payload.RepositoryID, "empty") + metrics.RecordJob("index", time.Since(start), true) + + return nil + } } branches = validBranches @@ -1111,7 +1181,15 @@ func (w *Worker) processSyncJob(ctx context.Context, job *queue.Job) error { adapter := &codeHostAdapter{client: client} - archivedRepos, err := w.reposService.SyncRepositories(ctx, payload.ConnectionID, adapter, conn.ExcludeArchived, cleanupArchived, conn.Repos, repoConfigs) + archivedRepos, err := w.reposService.SyncRepositories( + ctx, + payload.ConnectionID, + adapter, + conn.ExcludeArchived, + cleanupArchived, + conn.Repos, + repoConfigs, + ) if err != nil { metrics.RecordJob("sync", time.Since(start), false) tracing.RecordError(ctx, err) @@ -1526,7 +1604,10 @@ func (w *Worker) processCleanupJob(ctx context.Context, job *queue.Job) error { if strings.HasPrefix(baseName, pattern+"_v") || strings.Contains(baseName, "%2F"+pattern+"_v") || strings.Contains(baseName, "/"+pattern+"_v") || - strings.HasSuffix(strings.TrimSuffix(baseName, "_v"+extractVersionSuffix(baseName)), pattern) { + strings.HasSuffix( + strings.TrimSuffix(baseName, "_v"+extractVersionSuffix(baseName)), + pattern, + ) { shardFiles = append(shardFiles, shardFile) break } @@ -1806,6 +1887,33 @@ func (w *Worker) discoverBranches(ctx context.Context, repoPath string) ([]strin return branches, nil } +// detectDefaultBranch discovers the actual default branch of a repository. +// Uses discoverBranches and picks the best candidate (preferring main > master). +// Returns empty string if the repo has no branches (truly empty). +func (w *Worker) detectDefaultBranch(ctx context.Context, repoPath string) string { + discovered, err := w.discoverBranches(ctx, repoPath) + if err != nil || len(discovered) == 0 { + return "" + } + + // If discoverBranches returned only "HEAD" fallback, repo is likely empty + if len(discovered) == 1 && discovered[0] == "HEAD" { + return "" + } + + // Prefer common default branch names + for _, preferred := range []string{"main", "master"} { + for _, b := range discovered { + if b == preferred { + return b + } + } + } + + // Fall back to first discovered branch + return discovered[0] +} + // branchExists checks if a branch exists in the local git repository. // It checks both local and remote tracking branches. func (w *Worker) branchExists(ctx context.Context, repoPath, branch string) bool { @@ -1818,7 +1926,14 @@ func (w *Worker) branchExists(ctx context.Context, repoPath, branch string) bool } // Also check remote tracking branch - cmd = exec.CommandContext(ctx, "git", "rev-parse", "--verify", "--quiet", "refs/remotes/origin/"+branch) + cmd = exec.CommandContext( + ctx, + "git", + "rev-parse", + "--verify", + "--quiet", + "refs/remotes/origin/"+branch, + ) cmd.Dir = repoPath return cmd.Run() == nil @@ -1988,8 +2103,65 @@ func (w *Worker) FetchRepository( return nil } +// isRepoNotFoundError checks if a git error indicates the repository +// no longer exists on the remote (deleted, access revoked, etc.). +// These errors are permanent and should not be retried. +func isRepoNotFoundError(err error) bool { + if err == nil { + return false + } + + errStr := strings.ToLower(err.Error()) + + patterns := []string{ + "repository not found", + "could not read from remote repository", + "does not appear to be a git repository", + "the requested url returned error: 404", + "404 not found", + "project not found", + "the project you were looking for could not be found", + } + + for _, pattern := range patterns { + if strings.Contains(errStr, pattern) { + return true + } + } + + return false +} + +// handleRepoNotFound handles the case where a repository no longer exists on +// the remote code host. Marks the repo as excluded and returns a PermanentError +// to prevent retries. +func (w *Worker) handleRepoNotFound( + ctx context.Context, + payload queue.IndexPayload, + logger *zap.Logger, + start time.Time, + originalErr error, +) error { + logger.Warn("Repository not found on remote, marking as excluded", + zap.Error(originalErr), + ) + + if err := w.reposService.ExcludeRepository(ctx, payload.RepositoryID); err != nil { + logger.Warn("Failed to exclude deleted repository", zap.Error(err)) + } + + w.updateIndexStatusBestEffort(ctx, payload.RepositoryID, "failed") + metrics.RecordJob("index", time.Since(start), false) + metrics.RecordIndexFailure("repo_not_found") + tracing.RecordError(ctx, originalErr) + + return queue.NewPermanentError( + fmt.Errorf("repository not found on remote: %w", originalErr), + ) +} + // categorizeIndexFailure categorizes indexing failures for better observability. -// Returns one of: oom_killed, timeout, git_error, zoekt_error, unknown. +// Returns one of: repo_not_found, oom_killed, timeout, git_error, zoekt_error, unknown. func categorizeIndexFailure(err error) string { if err == nil { return "unknown" @@ -2012,6 +2184,11 @@ func categorizeIndexFailure(err error) string { return "timeout" } + // Check for repo not found (before generic git_error) + if isRepoNotFoundError(err) { + return "repo_not_found" + } + // Check for git errors if strings.Contains(errStr, "git") || strings.Contains(errStr, "clone") || diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 0526048..5cc7fa2 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -36,7 +36,8 @@ func NewDistributedLock(client *redis.Client, key string, ttl time.Duration) *Di // TryAcquire attempts to acquire the lock without blocking // Returns true if lock was acquired, false otherwise. func (l *DistributedLock) TryAcquire(ctx context.Context) (bool, error) { - _, err := l.client.SetArgs(ctx, l.key, l.workerID, redis.SetArgs{Mode: "NX", TTL: l.ttl}).Result() + _, err := l.client.SetArgs(ctx, l.key, l.workerID, redis.SetArgs{Mode: "NX", TTL: l.ttl}). + Result() if errors.Is(err, redis.Nil) { return false, nil } diff --git a/internal/queue/queue.go b/internal/queue/queue.go index 3cd0c38..dce30d5 100644 --- a/internal/queue/queue.go +++ b/internal/queue/queue.go @@ -146,6 +146,7 @@ type Queue struct { activeIndexKey string // SET of repo IDs with active index jobs activeSyncKey string // SET of connection IDs with active sync jobs activeCleanupKey string // SET of repo IDs with active cleanup jobs + processingKey string // SET of job IDs currently being processed // Sorted set indexes for efficient queries (score = unix timestamp in milliseconds) jobIndexKey string // All jobs sorted by creation time statusIndexPrefix string // Jobs by status: codesearch:jobs:status:{status} @@ -163,6 +164,7 @@ func NewQueue(client *redis.Client) *Queue { activeIndexKey: "codesearch:active:index", activeSyncKey: "codesearch:active:sync", activeCleanupKey: "codesearch:active:cleanup", + processingKey: "codesearch:jobs:processing", jobIndexKey: "codesearch:jobs:index", statusIndexPrefix: "codesearch:jobs:status:", typeIndexPrefix: "codesearch:jobs:type:", @@ -390,6 +392,32 @@ func (q *Queue) EnqueueWithOptions( // ErrJobAlreadyExists is returned when trying to enqueue a duplicate job. var ErrJobAlreadyExists = errors.New("job already exists") +// PermanentError wraps an error to indicate it should not be retried. +// When a job fails with a PermanentError, the retry mechanism skips +// all remaining attempts and marks the job as permanently failed. +type PermanentError struct { + Err error +} + +func (e *PermanentError) Error() string { + return e.Err.Error() +} + +func (e *PermanentError) Unwrap() error { + return e.Err +} + +// NewPermanentError creates a PermanentError wrapping the given error. +func NewPermanentError(err error) *PermanentError { + return &PermanentError{Err: err} +} + +// IsPermanentError checks if an error (or any error in its chain) is a PermanentError. +func IsPermanentError(err error) bool { + var pe *PermanentError + return errors.As(err, &pe) +} + // FindPendingJob searches for an existing pending or running job of the given type // that matches the provided key extractor function. // The keyExtractor is called for each job payload and should return a comparable key. @@ -960,7 +988,8 @@ func (q *Queue) ListJobsWithOptions( // Get all job IDs from the index (sorted by creation time, newest first) // ZRangeArgs with Rev returns in reverse order (highest score first = newest) - jobIDs, err := q.client.ZRangeArgs(ctx, redis.ZRangeArgs{Key: indexKey, Start: int64(0), Stop: int64(-1), Rev: true}).Result() + jobIDs, err := q.client.ZRangeArgs(ctx, redis.ZRangeArgs{Key: indexKey, Start: int64(0), Stop: int64(-1), Rev: true}). + Result() if err != nil { return nil, fmt.Errorf("get job index: %w", err) } @@ -1104,8 +1133,9 @@ func (q *Queue) QueueLength(ctx context.Context) (int64, error) { // CleanupResult contains the result of a cleanup operation. type CleanupResult struct { - DeletedCount int `json:"deleted_count"` - ScannedCount int `json:"scanned_count"` + DeletedCount int `json:"deleted_count"` + ScannedCount int `json:"scanned_count"` + GhostEntriesCount int `json:"ghost_entries_count"` } // CleanupOldJobs removes completed and failed jobs older than the specified duration. @@ -1159,17 +1189,20 @@ func (q *Queue) CleanupOldJobs(ctx context.Context, maxAge time.Duration) (*Clea switch job.Type { case JobTypeIndex: var payload IndexPayload - if err := json.Unmarshal(job.Payload, &payload); err == nil && payload.RepositoryID > 0 { + if err := json.Unmarshal(job.Payload, &payload); err == nil && + payload.RepositoryID > 0 { _ = q.MarkIndexJobInactive(ctx, payload.RepositoryID) } case JobTypeSync: var payload SyncPayload - if err := json.Unmarshal(job.Payload, &payload); err == nil && payload.ConnectionID > 0 { + if err := json.Unmarshal(job.Payload, &payload); err == nil && + payload.ConnectionID > 0 { _ = q.MarkSyncJobInactive(ctx, payload.ConnectionID) } case JobTypeCleanup: var payload CleanupPayload - if err := json.Unmarshal(job.Payload, &payload); err == nil && payload.RepositoryID > 0 { + if err := json.Unmarshal(job.Payload, &payload); err == nil && + payload.RepositoryID > 0 { _ = q.MarkCleanupJobInactive(ctx, payload.RepositoryID) } } @@ -1180,9 +1213,102 @@ func (q *Queue) CleanupOldJobs(ctx context.Context, maxAge time.Duration) (*Clea } } + // Clean up ghost entries in running/pending status indexes and type/main indexes. + // These accumulate when job data expires (24h TTL) but index entries persist. + result.GhostEntriesCount += q.cleanupGhostIndexEntries(ctx) + return result, nil } +// cleanupGhostIndexEntries removes entries from status, type, and main indexes +// where the underlying job data no longer exists (expired via Redis TTL). +// Returns the number of ghost entries removed. +func (q *Queue) cleanupGhostIndexEntries(ctx context.Context) int { + removed := 0 + + // Clean ghost entries from running and pending status indexes + for _, status := range []JobStatus{JobStatusRunning, JobStatusPending} { + indexKey := q.statusIndexKey(status) + + jobIDs, err := q.client.ZRange(ctx, indexKey, 0, -1).Result() + if err != nil { + continue + } + + for _, jobID := range jobIDs { + exists, err := q.client.Exists(ctx, q.jobPrefix+jobID).Result() + if err != nil { + continue + } + + if exists == 0 { + q.client.ZRem(ctx, indexKey, jobID) + + removed++ + } + } + } + + // Clean ghost entries from type indexes + for _, jobType := range []JobType{JobTypeIndex, JobTypeReplace, JobTypeSync, JobTypeCleanup} { + indexKey := q.typeIndexKey(jobType) + + jobIDs, err := q.client.ZRange(ctx, indexKey, 0, -1).Result() + if err != nil { + continue + } + + for _, jobID := range jobIDs { + exists, err := q.client.Exists(ctx, q.jobPrefix+jobID).Result() + if err != nil { + continue + } + + if exists == 0 { + q.client.ZRem(ctx, indexKey, jobID) + + removed++ + } + } + } + + // Clean ghost entries from the main job index + jobIDs, err := q.client.ZRange(ctx, q.jobIndexKey, 0, -1).Result() + if err == nil { + for _, jobID := range jobIDs { + exists, err := q.client.Exists(ctx, q.jobPrefix+jobID).Result() + if err != nil { + continue + } + + if exists == 0 { + q.client.ZRem(ctx, q.jobIndexKey, jobID) + + removed++ + } + } + } + + // Clean ghost entries from the processing set + processingIDs, err := q.client.SMembers(ctx, q.processingKey).Result() + if err == nil { + for _, jobID := range processingIDs { + exists, err := q.client.Exists(ctx, q.jobPrefix+jobID).Result() + if err != nil { + continue + } + + if exists == 0 { + q.client.SRem(ctx, q.processingKey, jobID) + + removed++ + } + } + } + + return removed +} + // Clear removes all jobs from the queue. func (q *Queue) Clear(ctx context.Context) error { err := q.client.Del(ctx, q.queueKey).Err() diff --git a/internal/queue/sharded_queue.go b/internal/queue/sharded_queue.go index d97379d..1e88d0d 100644 --- a/internal/queue/sharded_queue.go +++ b/internal/queue/sharded_queue.go @@ -58,7 +58,7 @@ func NewShardedQueue(client *redis.Client) *ShardedQueue { // processingKey returns the key for jobs being processed. func (sq *ShardedQueue) processingKey() string { - return "codesearch:jobs:processing" + return sq.Queue.processingKey } // workerKey returns the key for tracking worker heartbeats. @@ -83,7 +83,9 @@ func (sq *ShardedQueue) DequeueForShard(ctx context.Context, timeout time.Durati result, err := sq.client.RPop(ctx, sq.priorityQueueKey).Result() if err == nil && result != "" { job, err := sq.GetJob(ctx, result) - if err == nil && job != nil { + if err != nil || job == nil { + sq.cleanupGhostJob(ctx, result) + } else { // Priority jobs (replace) don't need shard checks - any worker can handle them if err := sq.claimJob(ctx, job); err == nil { return job, nil @@ -114,6 +116,10 @@ func (sq *ShardedQueue) DequeueForShard(ctx context.Context, timeout time.Durati job, err := sq.GetJob(ctx, jobID) if err != nil || job == nil { + // Job data expired — clean up stale index entries for this ghost job. + // We don't know the type/status, so remove from all possible indexes. + sq.cleanupGhostJob(ctx, jobID) + continue } @@ -149,7 +155,8 @@ func (sq *ShardedQueue) claimJob(ctx context.Context, job *Job) error { workerKey := sq.workerKey(job.ID) // SET NX is atomic - only succeeds if key doesn't exist - _, err := sq.client.SetArgs(ctx, workerKey, sq.workerID, redis.SetArgs{Mode: "NX", TTL: DefaultJobClaimTTL}).Result() + _, err := sq.client.SetArgs(ctx, workerKey, sq.workerID, redis.SetArgs{Mode: "NX", TTL: DefaultJobClaimTTL}). + Result() if errors.Is(err, redis.Nil) { return errors.New("job already claimed") } @@ -198,28 +205,32 @@ func (sq *ShardedQueue) MarkCompletedAndRelease(ctx context.Context, jobID strin // MarkFailedAndRelease marks a job failed and releases it. // If the job has retries remaining, it will be scheduled for retry instead of being marked as permanently failed. +// If the error is a PermanentError, retries are skipped entirely. func (sq *ShardedQueue) MarkFailedAndRelease( ctx context.Context, jobID string, jobErr error, ) error { - // Try to schedule retry - retried, err := sq.ScheduleRetry(ctx, jobID, jobErr) - if err != nil { - // If retry scheduling fails, fall back to marking as failed - if markErr := sq.MarkFailed(ctx, jobID, jobErr); markErr != nil { - return markErr + if IsPermanentError(jobErr) { + // Permanent errors skip retry — mark as failed immediately + _ = sq.MarkFailed(ctx, jobID, jobErr) + } else { + // Try to schedule retry + retried, err := sq.ScheduleRetry(ctx, jobID, jobErr) + if err != nil { + // If retry scheduling fails, fall back to marking as failed + if markErr := sq.MarkFailed(ctx, jobID, jobErr); markErr != nil { + return markErr + } } - } - // Release the job from processing regardless of retry status - releaseErr := sq.ReleaseJob(ctx, jobID) - - // If job was not retried (max attempts exceeded), it's already marked as failed by ScheduleRetry - // If job was retried, it's now in the retry queue waiting for its scheduled time - _ = retried // Used implicitly by ScheduleRetry updating the job + // If job was not retried (max attempts exceeded), it's already marked as failed by ScheduleRetry + // If job was retried, it's now in the retry queue waiting for its scheduled time + _ = retried // Used implicitly by ScheduleRetry updating the job + } - return releaseErr + // Release the job from processing regardless + return sq.ReleaseJob(ctx, jobID) } // RecoverStaleJobs finds jobs that were being processed but worker died @@ -343,3 +354,22 @@ func (sq *ShardedQueue) GetTotalShards() int { func (sq *ShardedQueue) IsShardingEnabled() bool { return sq.enabled } + +// cleanupGhostJob removes stale index entries for a job whose data has expired. +// Since we don't know the original type/status, we remove from all possible indexes. +func (sq *ShardedQueue) cleanupGhostJob(ctx context.Context, jobID string) { + pipe := sq.client.Pipeline() + pipe.ZRem(ctx, sq.jobIndexKey, jobID) + + for _, status := range []JobStatus{JobStatusPending, JobStatusRunning, JobStatusCompleted, JobStatusFailed} { + pipe.ZRem(ctx, sq.statusIndexKey(status), jobID) + } + + for _, jobType := range []JobType{JobTypeIndex, JobTypeReplace, JobTypeSync, JobTypeCleanup} { + pipe.ZRem(ctx, sq.typeIndexKey(jobType), jobID) + } + + pipe.SRem(ctx, sq.processingKey(), jobID) + + _, _ = pipe.Exec(ctx) +} diff --git a/internal/replace/service.go b/internal/replace/service.go index dcc3d12..b306e36 100644 --- a/internal/replace/service.go +++ b/internal/replace/service.go @@ -700,7 +700,16 @@ func (s *Service) cloneRepo( defer cancel() // Shallow clone — we only need the latest commit to apply changes and push a new branch - cmd := exec.CommandContext(cloneCtx, "git", "clone", "--depth", "1", "--single-branch", authURL, destDir) + cmd := exec.CommandContext( + cloneCtx, + "git", + "clone", + "--depth", + "1", + "--single-branch", + authURL, + destDir, + ) configureGitCmd(cmd) output, err := cmd.CombinedOutput() diff --git a/internal/repos/service.go b/internal/repos/service.go index 6b5e898..e5e4a80 100644 --- a/internal/repos/service.go +++ b/internal/repos/service.go @@ -922,6 +922,21 @@ func (s *Service) DeleteRepository(ctx context.Context, id int64) error { return nil } +// UpdateDefaultBranch updates the default branch for a repository. +// Called when the indexer detects a branch rename (e.g. master → main). +func (s *Service) UpdateDefaultBranch(ctx context.Context, id int64, branch string) error { + _, err := s.pool.Exec( + ctx, + `UPDATE repositories SET default_branch = $2, updated_at = NOW() WHERE id = $1`, + id, branch, + ) + if err != nil { + return fmt.Errorf("update default branch: %w", err) + } + + return nil +} + // ExcludeRepository marks a repository as excluded (soft delete) // Excluded repos are skipped during sync and should be removed from the index. func (s *Service) ExcludeRepository(ctx context.Context, id int64) error { @@ -1092,7 +1107,10 @@ func (s *Service) SyncRepositories( WHERE connection_id = $1 AND name = $2 `, connectionID, repo.FullName).Scan(&repoID, ¤tExcluded) if err == nil && !currentExcluded { - archivedRepos = append(archivedRepos, ArchivedRepo{ID: repoID, Name: repo.FullName}) + archivedRepos = append( + archivedRepos, + ArchivedRepo{ID: repoID, Name: repo.FullName}, + ) } } @@ -1136,7 +1154,8 @@ func (s *Service) SyncRepositories( // Determine initial deleted/excluded state from config (only applies to NEW repos) initialDeleted := hasConfig && repoConfig.Delete - initialExcluded := hasConfig && (repoConfig.Exclude || repoConfig.Delete) // deleted repos are also excluded + initialExcluded := hasConfig && + (repoConfig.Exclude || repoConfig.Delete) // deleted repos are also excluded if isMySQL { // MySQL: use INSERT ... ON DUPLICATE KEY UPDATE diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 9630f49..4629670 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -111,8 +111,11 @@ func New( redisClient: redisClient, reposService: repos.NewService(pool, tokenEncryptor), tokenEncryptor: tokenEncryptor, - logger: logger.With(zap.String("component", "scheduler"), zap.String("instance", instanceID)), - instanceID: instanceID, + logger: logger.With( + zap.String("component", "scheduler"), + zap.String("instance", instanceID), + ), + instanceID: instanceID, } } @@ -174,7 +177,8 @@ func (s *Scheduler) Stop() { // tryBecomeLeader attempts to acquire the scheduler leader lock. // Returns true if this instance is now the leader. func (s *Scheduler) tryBecomeLeader(ctx context.Context) bool { - _, err := s.redisClient.SetArgs(ctx, leaderKey, s.instanceID, redis.SetArgs{Mode: "NX", TTL: leaderLeaseTTL}).Result() + _, err := s.redisClient.SetArgs(ctx, leaderKey, s.instanceID, redis.SetArgs{Mode: "NX", TTL: leaderLeaseTTL}). + Result() if errors.Is(err, redis.Nil) { return false } @@ -322,10 +326,11 @@ func (s *Scheduler) cleanupOldJobs(ctx context.Context) { return } - if result.DeletedCount > 0 { + if result.DeletedCount > 0 || result.GhostEntriesCount > 0 { s.logger.Info("Cleaned up old jobs", zap.Int("deleted", result.DeletedCount), zap.Int("scanned", result.ScannedCount), + zap.Int("ghost_entries_removed", result.GhostEntriesCount), zap.Duration("retention", s.cfg.JobRetentionPeriod), ) } else { @@ -377,7 +382,10 @@ func (s *Scheduler) recoverOrphanedActiveJobs(ctx context.Context) { return } - s.logger.Debug("Checking active index jobs for orphans", zap.Int("active_count", len(activeRepos))) + s.logger.Debug( + "Checking active index jobs for orphans", + zap.Int("active_count", len(activeRepos)), + ) // Check each repo to see if it has an actual pending/running job orphanCount := 0 @@ -554,7 +562,11 @@ func (s *Scheduler) cleanupOrphanShards(ctx context.Context) { // cleanupOrphanSCIPDatabases removes SCIP database files whose repo IDs no // longer map to a valid (non-excluded) repository. -func (s *Scheduler) cleanupOrphanSCIPDatabases(ctx context.Context, _ map[string]bool, allRepos []repos.Repository) { +func (s *Scheduler) cleanupOrphanSCIPDatabases( + ctx context.Context, + _ map[string]bool, + allRepos []repos.Repository, +) { scipIDs, err := s.scipService.ListIndexedRepoIDs() if err != nil { s.logger.Warn("Failed to list SCIP indexed repo IDs", zap.Error(err)) diff --git a/internal/scip/federated.go b/internal/scip/federated.go index c719a63..2c8a5c1 100644 --- a/internal/scip/federated.go +++ b/internal/scip/federated.go @@ -43,7 +43,11 @@ func (c *FederatedClient) getShardURL(shard int) string { } // HasIndex checks if a SCIP index exists for a repo on its owning shard. -func (c *FederatedClient) HasIndex(ctx context.Context, repoName string, repoID int64) (bool, error) { +func (c *FederatedClient) HasIndex( + ctx context.Context, + repoName string, + repoID int64, +) (bool, error) { shard := c.getShardForRepo(repoName) shardURL := c.getShardURL(shard) @@ -162,7 +166,11 @@ func (c *FederatedClient) FindReferences( } // GetStats proxies a stats request to the owning shard. -func (c *FederatedClient) GetStats(ctx context.Context, repoName string, repoID int64) (map[string]any, error) { +func (c *FederatedClient) GetStats( + ctx context.Context, + repoName string, + repoID int64, +) (map[string]any, error) { shard := c.getShardForRepo(repoName) shardURL := c.getShardURL(shard) diff --git a/internal/scip/indexer.go b/internal/scip/indexer.go index 9b81218..d1e4769 100644 --- a/internal/scip/indexer.go +++ b/internal/scip/indexer.go @@ -405,7 +405,11 @@ func scipLanguages() map[string]struct { // returned, including root, because each marker is an independent project. // // If no markers are found, returns a single entry with the repo root and empty prefix. -func (i *Indexer) findAllProjectDirs(repoPath string, markerFiles []string, independentDirs bool) []projectLocation { +func (i *Indexer) findAllProjectDirs( + repoPath string, + markerFiles []string, + independentDirs bool, +) []projectLocation { // Check if any marker exists at the root rootHasMarker := false @@ -551,7 +555,10 @@ func (i *Indexer) indexGoProject(ctx context.Context, projectDir string) ([]byte } // indexTypeScriptProject indexes a single TypeScript/JavaScript project at the given directory. -func (i *Indexer) indexTypeScriptProject(ctx context.Context, projectDir string) ([]byte, string, error) { +func (i *Indexer) indexTypeScriptProject( + ctx context.Context, + projectDir string, +) ([]byte, string, error) { // Check for scip-typescript via npx scipTS := i.config.SCIPTypeScript if scipTS == "" { @@ -689,7 +696,10 @@ func (i *Indexer) indexRustProject(ctx context.Context, projectDir string) ([]by } // indexPythonProject indexes a single Python project at the given directory. -func (i *Indexer) indexPythonProject(ctx context.Context, projectDir string) ([]byte, string, error) { +func (i *Indexer) indexPythonProject( + ctx context.Context, + projectDir string, +) ([]byte, string, error) { // Check for scip-python scipPython := i.config.SCIPPython if scipPython == "" { @@ -763,7 +773,11 @@ func (i *Indexer) indexPHPProject( files = append(files, e.Name()) } - i.logger.Debug("Files in project path", zap.Strings("files", files), zap.String("projectDir", projectDir)) + i.logger.Debug( + "Files in project path", + zap.Strings("files", files), + zap.String("projectDir", projectDir), + ) // scip-php requires composer.json - check if it exists composerJSON := filepath.Join(projectDir, "composer.json") @@ -815,7 +829,10 @@ func (i *Indexer) indexPHPProject( } // Run composer install to get project dependencies - i.logger.Info("Running composer install for PHP SCIP indexing", zap.String("projectDir", projectDir)) + i.logger.Info( + "Running composer install for PHP SCIP indexing", + zap.String("projectDir", projectDir), + ) installCmd := exec.CommandContext( ctx, @@ -834,7 +851,11 @@ func (i *Indexer) indexPHPProject( installCmd.Stderr = &installOut if err := installCmd.Run(); err != nil { - i.logger.Warn("composer install failed", zap.Error(err), zap.String("output", installOut.String())) + i.logger.Warn( + "composer install failed", + zap.Error(err), + zap.String("output", installOut.String()), + ) } // Check if scip-php is installed in the project @@ -859,7 +880,11 @@ func (i *Indexer) indexPHPProject( cmd.Stdout = &stdout cmd.Stderr = &stderr - i.logger.Info("Running scip-php", zap.String("projectDir", projectDir), zap.String("scipPHP", scipPHP)) + i.logger.Info( + "Running scip-php", + zap.String("projectDir", projectDir), + zap.String("scipPHP", scipPHP), + ) if err := cmd.Run(); err != nil { output := stderr.String() + stdout.String() diff --git a/internal/scip/service.go b/internal/scip/service.go index e1c6209..0e2eb40 100644 --- a/internal/scip/service.go +++ b/internal/scip/service.go @@ -42,7 +42,11 @@ func NewService(cacheDir string, logger *zap.Logger) (*Service, error) { // NewServiceWithConfig creates a new SCIP service with an explicit IndexerConfig. // This allows callers to configure indexer binary paths and timeouts. -func NewServiceWithConfig(cacheDir string, indexerCfg IndexerConfig, logger *zap.Logger) (*Service, error) { +func NewServiceWithConfig( + cacheDir string, + indexerCfg IndexerConfig, + logger *zap.Logger, +) (*Service, error) { if err := os.MkdirAll(cacheDir, 0o755); err != nil { return nil, err } diff --git a/internal/tracing/middleware_test.go b/internal/tracing/middleware_test.go index f44a5c6..883b4c5 100644 --- a/internal/tracing/middleware_test.go +++ b/internal/tracing/middleware_test.go @@ -16,7 +16,12 @@ func TestHTTPMiddleware(t *testing.T) { wrapped := HTTPMiddleware(handler) - req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/api/repos", nil) + req := httptest.NewRequestWithContext( + context.Background(), + http.MethodGet, + "/api/repos", + nil, + ) rec := httptest.NewRecorder() wrapped.ServeHTTP(rec, req) @@ -38,7 +43,12 @@ func TestHTTPMiddleware(t *testing.T) { wrapped := HTTPMiddleware(handler) - req := httptest.NewRequestWithContext(context.Background(), http.MethodGet, "/api/notfound", nil) + req := httptest.NewRequestWithContext( + context.Background(), + http.MethodGet, + "/api/notfound", + nil, + ) rec := httptest.NewRecorder() wrapped.ServeHTTP(rec, req) @@ -55,7 +65,12 @@ func TestHTTPMiddleware(t *testing.T) { wrapped := HTTPMiddleware(handler) - req := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/api/repos", nil) + req := httptest.NewRequestWithContext( + context.Background(), + http.MethodPost, + "/api/repos", + nil, + ) rec := httptest.NewRecorder() wrapped.ServeHTTP(rec, req) diff --git a/web/.prettierignore b/web/.prettierignore new file mode 100644 index 0000000..dd27691 --- /dev/null +++ b/web/.prettierignore @@ -0,0 +1,5 @@ +.next/ +node_modules/ +out/ +coverage/ +bun.lock diff --git a/web/.prettierrc b/web/.prettierrc new file mode 100644 index 0000000..35bfe79 --- /dev/null +++ b/web/.prettierrc @@ -0,0 +1,7 @@ +{ + "semi": true, + "singleQuote": false, + "tabWidth": 2, + "trailingComma": "es5", + "plugins": ["prettier-plugin-tailwindcss"] +} diff --git a/web/app/HomeClient.tsx b/web/app/HomeClient.tsx index 5e0ebfd..c5fc7ea 100644 --- a/web/app/HomeClient.tsx +++ b/web/app/HomeClient.tsx @@ -12,45 +12,56 @@ function HomeContent() { const [repoCount, setRepoCount] = useState(null); useEffect(() => { - api.listRepos().then((response) => setRepoCount(response.total_count)).catch(() => { }); + api + .listRepos() + .then((response) => setRepoCount(response.total_count)) + .catch(() => {}); }, []); return (
-
-
+
+
{/* Header Section */} -
+
{!results && !loading ? ( // Hero state - centered -
+
- -

+ +

Code Search

-

- Search across {repoCount !== null ? ( - {repoCount} repositories +

+ Search across{" "} + {repoCount !== null ? ( + + {repoCount} repositories + ) : ( "your repositories" - )} with Zoekt + )}{" "} + with Zoekt

) : ( // Results state - left aligned (consistent with other pages) <>
- -

Search Results

+ +

+ Search Results +

)} @@ -61,22 +72,35 @@ function HomeContent() { {/* Search Help - show when no results and not loading */} {!loading && !results && (
-

+

How to search

-
+
{/* Left Column - Search in files */}
-

+

Search patterns

-
- - - +
+ + FOO case:yes} + query="foo or bar" + description="either foo or bar" + /> + + FOO case:yes + + } description="case sensitive" />
@@ -84,24 +108,40 @@ function HomeContent() { {/* Right Column - Filter results */}
-

+

Filter results

-
+
lang:go} + query={ + <> + lang:go + + } description="by language" /> file:README} + query={ + <> + file:README + + } description="by filename" /> repo:org/repo} + query={ + <> + repo:org/repo + + } description="by repository" /> content:foo} + query={ + <> + content:foo + + } description="search content only" />
@@ -109,24 +149,42 @@ function HomeContent() { {/* Advanced section */}
-

+

Advanced

-
+
foo -lang:go} + query={ + <> + foo -lang:go + + } description="negate filter" /> sym:GetFoo} + query={ + <> + sym:GetFoo + + } description="symbol search" /> file:\.ts$} + query={ + <> + file:\.ts$ + + } description='files that end in ".ts"' /> /foo-(bar|baz)/} + query={ + <> + + /foo-(bar|baz)/ + + + } description="regular expression" />
@@ -134,12 +192,12 @@ function HomeContent() {
{/* Full documentation link */} -
+
View full query syntax documentation → @@ -149,14 +207,19 @@ function HomeContent() { {/* Show loading indicator only when no results yet */} {loading && !results && ( -
- -

Searching...

+
+ +

+ Searching... +

)} {/* Always show results if available (supports streaming) */} - +
@@ -165,11 +228,13 @@ function HomeContent() { export default function HomeClient() { return ( - - -
- }> + + +
+ } + > ); @@ -177,15 +242,19 @@ export default function HomeClient() { function SearchExample({ query, - description + description, }: { query: React.ReactNode; description: string; }) { return ( -
- {query} - ({description}) +
+ + {query} + + + ({description}) +
); } diff --git a/web/app/connections/ConnectionsClient.tsx b/web/app/connections/ConnectionsClient.tsx index 2e7b11a..aa495cf 100644 --- a/web/app/connections/ConnectionsClient.tsx +++ b/web/app/connections/ConnectionsClient.tsx @@ -31,7 +31,7 @@ function ConnectionActionMenu({ onEdit, onDelete, syncing, - readonly + readonly, }: { connection: Connection; onSync: (id: number) => void; @@ -58,26 +58,28 @@ function ConnectionActionMenu({
{isOpen && ( -
+
{!readonly && ( <> @@ -86,9 +88,9 @@ function ConnectionActionMenu({ onEdit(connection); setIsOpen(false); }} - className="w-full flex items-center gap-2 px-3 py-2 text-sm text-left hover:bg-gray-50 dark:hover:bg-gray-700 transition-colors" + className="flex w-full items-center gap-2 px-3 py-2 text-left text-sm transition-colors hover:bg-gray-50 dark:hover:bg-gray-700" > - + Edit @@ -147,27 +149,35 @@ function ConnectionForm({ formExcludeArchived: boolean; submitting: boolean; editingConnection: Connection | null; - updateForm: (update: Partial<{ - showForm: boolean; - formName: string; - formType: string; - formUrl: string; - formToken: string; - formExcludeArchived: boolean; - submitting: boolean; - editingConnection: Connection | null; - }>) => void; + updateForm: ( + update: Partial<{ + showForm: boolean; + formName: string; + formType: string; + formUrl: string; + formToken: string; + formExcludeArchived: boolean; + submitting: boolean; + editingConnection: Connection | null; + }> + ) => void; getDefaultUrl: (type: string) => string; onSubmit: (e: React.FormEvent) => void; }) { return ( -
-

+ +

{editingConnection ? "Edit Connection" : "New Connection"}

-
+
-
-
-
-