diff --git a/internal/context/repomap/repomap.go b/internal/context/repomap/repomap.go index 5ed88780..4638166f 100644 --- a/internal/context/repomap/repomap.go +++ b/internal/context/repomap/repomap.go @@ -1,13 +1,35 @@ -// Package repomap builds an AST-ranked overview of a codebase within a token -// budget, in the spirit of Aider's repository map. +// Package repomap is the prompt-injection shim that produces a token-budgeted +// repository overview for hawk's context layer. It builds an import/refer +// graph over the source files in root, ranks the nodes with a PageRank +// pass, and renders the highest-ranked files (and their top symbols) as a +// compact text block capped at Options.Budget tokens. // -// It constructs a graph where files are nodes and edges are import/reference -// dependencies, ranks the nodes with a PageRank-like pass, and emits a compact -// textual map (file -> key symbols) constrained to a configurable token budget. +// # Relationship to internal/intelligence/repomap // -// The package is intentionally dependency-light: Go files are parsed with -// go/parser + go/ast; other languages fall back to a small regex extractor. -// Only the standard library is used. +// hawk ships a second package, internal/intelligence/repomap, that exposes +// a much larger surface: call graphs, search, quality signals, API +// scanning, incremental indexing, and so on. That package is the deep +// analysis engine. THIS package is intentionally narrow: one entry point +// (RepoMap), a single Graph type, a self-contained scan/rank/render +// pipeline, and a stdlib-only dependency surface. The two packages share +// no code; they merely share a name and a goal (a useful map of a +// repository). The deep package's doc.go spells this out from the other +// side of the boundary. +// +// Callers that need more than a budgeted text block - symbol-level +// navigation, BM25 search, dead-code detection, OpenAPI export, etc. - +// should import internal/intelligence/repomap directly instead of +// extending this one. +// +// # Implementation notes +// +// Go files are parsed with go/parser and go/ast; other languages fall +// back to a small regex extractor. Only the standard library is used. +// Files larger than 1 MiB, hidden directories, and a small set of +// well-known build/vendor trees are skipped during the scan. The +// PageRank pass uses the standard damped iteration with dangling-mass +// redistribution and exits early once the total node-to-node delta +// drops below 1e-9. package repomap import ( diff --git a/internal/intelligence/repomap/README.md b/internal/intelligence/repomap/README.md new file mode 100644 index 00000000..60ca7c53 --- /dev/null +++ b/internal/intelligence/repomap/README.md @@ -0,0 +1,288 @@ +# `internal/intelligence/repomap/` + +> Deep code-analysis engine for hawk: language-aware symbol extraction, +> static analysis, search, quality signals, API scanning, and incremental +> indexing. Distinct from `internal/context/repomap`, which is the narrow +> prompt-injection shim used by the context layer. + +## What it does + +`Generate(dir, opts)` walks a directory, dispatches each supported source +file to a language-aware parser, and returns a `RepoMap`: a token-budgeted +summary of files and their top-level symbols suitable for injection into +LLM prompts. Around that core the package accumulates a large set of +specialised analyses - call graph, import graph, type hierarchy, code +ownership, BM25 search, cyclomatic complexity, code smells, dead-code +detection, health score, doc linter, migration detector, HTTP route +scanner, and an incremental file-hash cache - that all share the same +parsed-symbol substrate. + +The package is stdlib-only at its core (`go/parser`, `go/ast`, `go/token`, +`encoding/*`). The only third-party dependency is `github.com/fsnotify +/fsnotify` for file watching; hawk's `internal/scoring` and +`internal/ui/icons` are pulled in where they are used. Tree-sitter is +deliberately not required: Go is parsed with `go/ast` and other languages +are handled by an enhanced regex extractor with scope tracking. + +## Architecture + +```mermaid +flowchart TB + subgraph Entry["Entry point"] + REPOMAP[repomap.go
Generate, RepoMap, Options] + end + + subgraph Core["Core"] + CACHE[cache.go
in-process LRU] + WATCHER[watcher.go
fsnotify wrapper] + GITIGNORE[gitignore.go
composed rules] + PATTERNS[patterns.go
include/exclude loader] + end + + subgraph Symbols["Symbols / parsing"] + PARSER[parser.go
regex-Go] + ENHANCED[parser_enhanced.go
AST-Go] + LANGS[parser_langs.go
regex non-Go] + TS[treesitter.go
scope-aware] + end + + subgraph Static["Static analysis"] + CALL[callgraph.go] + DEP[depgraph.go] + IMP[imports.go] + HIER[hierarchy.go] + IFACE[interface_extract.go] + COCHG[cochange.go] + CHG[changeset.go] + OWN[ownership.go] + SHAP[shapley.go] + end + + subgraph Search["Search / navigation"] + NAV[navigation.go] + SEM[semantic.go] + SEMSRCH[semantic_search.go] + RERANK[rerank.go] + PR[pagerank.go] + PRED[predict.go] + end + + subgraph Quality["Quality signals"] + CPLX[complexity.go] + SMELL[smells.go] + HEALTH[health_score.go] + DOC[doclint.go] + DEAD[dead_code.go] + MIG[migration_detector.go] + end + + subgraph API["API surface"] + SCAN[api_scanner.go] + end + + subgraph Incr["Incremental"] + INCR[incremental.go] + INCRM[incremental_map.go] + end + + subgraph Group["Grouping"] + GROUPER[file_grouper.go] + SUMMARY[summary.go] + end + + REPOMAP --> CACHE + REPOMAP --> WATCHER + REPOMAP --> GITIGNORE + REPOMAP --> PATTERNS + REPOMAP --> PARSER + REPOMAP --> ENHANCED + REPOMAP --> LANGS + REPOMAP --> TS + Static --> IMP + Static --> CALL + Search --> PR + Search --> SEM + Search --> SEMSRCH + Quality --> CPLX + Quality --> SMELL + Quality --> HEALTH + Quality --> DOC + Quality --> DEAD + Quality --> MIG + API --> SCAN + Incr --> INCR + Incr --> INCRM + Group --> GROUPER + Group --> SUMMARY +``` + +## File groups + +| Group | Files | Purpose | +|------------------------|--------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------| +| **Core** | `repomap.go`, `cache.go`, `watcher.go`, `gitignore.go`, `patterns.go` | Entry point, file scanning, file watching, in-process and persistent caches | +| **Symbols / parsing** | `parser.go`, `parser_enhanced.go`, `parser_langs.go`, `treesitter.go` | Language-aware symbol extraction: regex Go, AST Go, regex non-Go, scope-aware (tree-sitter-like) | +| **Static analysis** | `callgraph.go`, `depgraph.go`, `imports.go`, `hierarchy.go`, `interface_extract.go`, `cochange.go`, `changeset.go`, `ownership.go`, `shapley.go` | Callers/callees, package-level deps, import graph, type hierarchy, exported surface, git-history co-change, change-set context, ownership, Shapley value ranker | +| **Search / navigation**| `navigation.go`, `semantic.go`, `semantic_search.go`, `rerank.go`, `pagerank.go`, `predict.go` | LSP-free navigation index, BM25 search, PageRank ranking, reranking, relevance prediction | +| **Quality signals** | `complexity.go`, `smells.go`, `health_score.go`, `doclint.go`, `dead_code.go`, `migration_detector.go` | Cyclomatic complexity, code smells, health rollup, doc linter, dead code, deprecated-API detection | +| **API surface** | `api_scanner.go` | HTTP route scanners (Chi, net/http, Gin, Echo, Gorilla, Fiber) + OpenAPI export | +| **Incremental** | `incremental.go`, `incremental_map.go` | `CodeIndexer` interface and reindex loop, persistent on-disk symbol cache | +| **Grouping** | `file_grouper.go`, `summary.go` | File grouping, codebase summary suitable for prompt injection | + +## Entry points + +The full public surface is read from the source. The headline entry points +are: + +- **`Generate(dir string, opts Options) (*RepoMap, error)`** in `repomap.go` - + the canonical entry point. Walks `dir`, parses every supported file, + and returns a `RepoMap` with `Files` and a `TokenEst`. +- **`(*RepoMap).Format(maxTokens int) string`** in `repomap.go` - renders + the map as text, truncating to fit `maxTokens`. +- **`BuildCallGraph(root string) (*CallGraph, error)`** in `callgraph.go` - + Go-only caller/callee graph from `go/ast`. +- **`BuildDepGraph` / `NewDepGraph` / `BuildFromGoMod` / `BuildFromPackageJSON`** + in `depgraph.go` - Go and JS/TS dependency graphs. +- **`BuildImportGraph(root string) (*ImportGraph, error)`** in `imports.go` - + file-level import graph. +- **`NewNavIndex` / `(*NavIndex).BuildIndex` / `(*NavIndex).GoToDefinition` / + `(*NavIndex).FindReferences` / `(*NavIndex).FindImplementations`** in + `navigation.go` - the LSP-free navigation API. +- **`BuildSemanticIndex` / `(*SemanticIndex).Search` / + `NewSemanticSearchIndex`** in `semantic.go` / `semantic_search.go` - + chunked TF-IDF and BM25 search. +- **`BuildSymbolGraph` / `(*SymbolGraph).TopSymbols`** in `pagerank.go` - + symbol-level PageRank. +- **`NewComplexityAnalyzer` / `(*ComplexityAnalyzer).FindHotspots`** in + `complexity.go` - complexity hotspots. +- **`NewSmellDetector` / `(*SmellDetector).ScanDirectory`** in `smells.go` + - code smell detection. +- **`NewHealthScorer` / `(*HealthScorer).Score` / `FormatScore` / + `CompareScores`** in `health_score.go` - health score rollup and + before/after diff. +- **`NewDeadCodeDetector` / `(*DeadCodeDetector).Detect`** in + `dead_code.go` - dead-code detection. +- **`NewMigrationDetector` / `(*MigrationDetector).Scan` / + `FormatOpportunities` / `AutoFix`** in `migration_detector.go` - + deprecated-API migration. +- **`NewAPIScanner` / `(*APIScanner).Scan` / `FormatAPIMap` / + `GenerateOpenAPI`** in `api_scanner.go` - HTTP route scanner. +- **`NewIncrementalMap(cacheDir string) (*IncrementalMap, error)`** in + `incremental_map.go` - persistent on-disk symbol cache. +- **`IncrementalReindex(dir string, ignore []string, indexer CodeIndexer) + (added, skipped, removed int, err error)`** in `incremental.go` - the + diff-and-reindex loop for the `CodeIndexer` interface. +- **`NewFileWatcher(root string, onChange func(path string)) + (*FileWatcher, error)`** in `watcher.go` - fsnotify wrapper. +- **`NewSummaryGenerator(projectDir string, maxTokens int) + *SummaryGenerator` / `RenderForPrompt` / `RenderCompact`** in + `summary.go` - the prompt-injectable codebase summary. +- **`BuildHierarchy(root string) (*HierarchicalSummary, error)`** in + `hierarchy.go` - 3-level project summary. +- **`PredictRelevantFiles` / `NewRecentEditTracker`** in `predict.go` - + relevance prediction from prompt, recent edits, import graph, and + symbol map. +- **`BuildCoChangeAnalysis(root string, commitLimit int) + (*CoChangeAnalysis, error)`** in `cochange.go` - git-history co-change. +- **`FromGitDiff` / `FromGitDiffRange`** in `changeset.go` - change-set + context. +- **`NewOwnershipMap` / `(*OwnershipMap).Compute`** in `ownership.go` - + per-file ownership. +- **`NewShapleyRanker(chunks []CodeChunk) *ShapleyRanker`** in `shapley.go` + - Shapley-value chunk ranking. +- **`NewAPIScanner`** in `api_scanner.go` - HTTP route scanner factory. + +## Storage model + +- **In-process symbol cache** (`cache.go`): an LRU keyed by + `(path, modtime)` capped at `defaultMaxSymbolCacheEntries` (5000). It + is consulted by `parseFileSymbols` in `repomap.go` and is cleared on + process exit. +- **Persistent incremental cache** (`incremental_map.go`): JSON file at + `/repomap-cache.json` (typically `.hawk/repomap-cache.json`) + keyed by SHA-256 of file content. `IncrementalReindex` diffs the + project tree against the cached hash set and re-parses only changed + files. +- **Watch protocol** (`watcher.go`): `NewFileWatcher(root, onChange)` walks + the tree, registers every non-hidden, non-vendor directory with + `fsnotify.Watcher`, and invokes `onChange(path)` on + `Write`/`Create`/`Remove` events for supported source files. `Start` + launches the event loop goroutine; `Stop` terminates it. + +## Extension points + +### Add a new language parser + +1. Add the extension to `isSupportedExt` in `repomap.go` so the walker + picks up the new files. +2. Add a new case to `parseFileSymbols` in `repomap.go` that dispatches + to a `parseX` function. +3. Add the `parseX(src string) []Symbol` function. For most languages + you can copy the `jsSpec` / `cSpec` patterns in `parser_langs.go`. +4. If the language needs scope-aware extraction, add a new + `TreeSitterParser` method in `treesitter.go` instead. +5. Optionally wire the same extension into `detectLang` in + `internal/context/repomap/scan.go` if the prompt-injection shim + should also pick it up. + +### Add a new code smell + +1. Add a `Detector func(...) []CodeSmell` field on `SmellDetector` in + `smells.go`. +2. Wire the new field into `NewSmellDetector` and `ScanDirectory`. +3. Tune `SmellThresholds` defaults if the new smell has tunable limits. + +### Add a new HTTP framework scanner + +1. Add a `ScanX(content, file string) []APIEndpoint` function in + `api_scanner.go` using one of the existing scanners (e.g. `ScanChi`) + as a template. +2. Add the corresponding case to `DetectFramework` so the dispatcher + knows to use the new scanner. +3. If the new framework uses a different routing style, update + `FormatAPIMap` and `GenerateOpenAPI` to handle the new metadata. + +## Performance and scaling + +- **`Generate`** is O(N) in the number of files with a hard cap on + `Options.MaxFiles` (default 500). The walk is single-threaded; the + per-file parsing is also single-threaded but the work is bounded per + file. +- **Symbol cache** (`cache.go`) keeps hot files in memory; it is + process-local and does not survive a restart. +- **IncrementalMap** (`incremental_map.go`) persists hashes and symbol + lists on disk. `IncrementalReindex` only re-parses files whose SHA-256 + has changed. For very large repositories (tens of thousands of files) + prefer the incremental path over `Generate`. +- **Static-analysis passes** (`callgraph`, `depgraph`, `pagerank`, + `shapley`) are O(V + E) per iteration over the symbol graph and scale + linearly with the number of declarations, not the number of lines. +- **BM25 search** (`semantic_search.go`) is O(Q * D) per query, where Q + is the number of query terms and D is the number of indexed documents. + IDF and average document length are precomputed and cached. +- **Health score** (`health_score.go`) is O(F) per dimension (F = file + count) and runs all dimensions in sequence; for very large projects + the per-file scans are the bottleneck. +- **Tree-sitter path** is not used. The "tree-sitter-style" scope-aware + extractor in `treesitter.go` is a pure-Go regex implementation that + avoids the CGO and binary dependencies of the real library. + +## Relationship to `internal/context/repomap` + +`internal/context/repomap` is a much narrower package - essentially just +`RepoMap(root, budget) (string, error)`. It is the prompt-injection shim +that hawk's context layer calls when it needs a budgeted overview for the +system prompt. It does its own AST parsing, PageRank pass, and rendering +and shares no code with this package beyond the name. + +Callers that need more than a budgeted text block (symbol-level +navigation, BM25 search, dead-code detection, OpenAPI export, etc.) +should import `internal/intelligence/repomap` (this package) directly. +See the comment at the top of `internal/context/repomap/repomap.go` for +the other side of the boundary. + +## See also + +- `doc.go` - go-doc compatible package overview. +- `doc_test.go` - worked example (calls `Generate` + `Format`). +- `internal/context/repomap/doc.go` - the shim's perspective. diff --git a/internal/intelligence/repomap/api_scanner.go b/internal/intelligence/repomap/api_scanner.go index 0b4c2755..1d7c54d5 100644 --- a/internal/intelligence/repomap/api_scanner.go +++ b/internal/intelligence/repomap/api_scanner.go @@ -1,3 +1,8 @@ +// api_scanner.go discovers HTTP endpoints in a project +// by detecting the framework (Chi, net/http, Gin, Echo, Gorilla mux, or +// Fiber) and extracting route declarations into an APIMap. FormatAPIMap +// renders the routes as text; GenerateOpenAPI produces a 3.x OpenAPI +// document for the same set. package repomap import ( diff --git a/internal/intelligence/repomap/cache.go b/internal/intelligence/repomap/cache.go index 9c00160f..8159e36f 100644 --- a/internal/intelligence/repomap/cache.go +++ b/internal/intelligence/repomap/cache.go @@ -1,3 +1,6 @@ +// cache.go implements the in-process LRU symbol cache keyed +// by (path, modtime). It is consulted by parseFileSymbols before re-parsing +// and is cleared on process exit; for a persistent cache, use IncrementalMap. package repomap import ( diff --git a/internal/intelligence/repomap/callgraph.go b/internal/intelligence/repomap/callgraph.go index 448b1315..da8ee178 100644 --- a/internal/intelligence/repomap/callgraph.go +++ b/internal/intelligence/repomap/callgraph.go @@ -1,3 +1,7 @@ +// callgraph.go builds a per-function caller/callee graph +// from a Go tree using go/parser and go/ast. It supports bounded BFS +// traversals in both directions (CallersOf, CalleesOf) and a symmetric +// Neighborhood helper used by the navigation index. package repomap import ( diff --git a/internal/intelligence/repomap/changeset.go b/internal/intelligence/repomap/changeset.go index 08db9d6f..d0ddb544 100644 --- a/internal/intelligence/repomap/changeset.go +++ b/internal/intelligence/repomap/changeset.go @@ -1,3 +1,8 @@ +// changeset.go derives a focused ChangeSetContext from +// a git diff (working tree or a range against a base ref). The context +// lists changed files, files affected by the changes (dependents), and +// files needed to understand the changes (imports), and is used to build +// smaller, change-set-aware LLM prompts. package repomap import ( diff --git a/internal/intelligence/repomap/cochange.go b/internal/intelligence/repomap/cochange.go index c2126352..1282d38d 100644 --- a/internal/intelligence/repomap/cochange.go +++ b/internal/intelligence/repomap/cochange.go @@ -1,3 +1,7 @@ +// cochange.go mines the last N git commits to build a +// co-occurrence matrix of files that change together, and exposes +// RelatedFiles for "what else should I look at" recommendations after a +// file has been touched. package repomap import ( diff --git a/internal/intelligence/repomap/complexity.go b/internal/intelligence/repomap/complexity.go index c6382a77..62788560 100644 --- a/internal/intelligence/repomap/complexity.go +++ b/internal/intelligence/repomap/complexity.go @@ -1,3 +1,9 @@ +// complexity.go computes cyclomatic and cognitive +// complexity per function (Go files via go/ast, other languages via +// brace-tracking regex), aggregates a ComplexityReport per file, and +// surfaces refactoring suggestions and a MaintainabilityIndex for the +// health score. FindHotspots walks an entire project and returns the +// highest-complexity functions first. package repomap import ( diff --git a/internal/intelligence/repomap/dead_code.go b/internal/intelligence/repomap/dead_code.go index 15c5a685..397ce040 100644 --- a/internal/intelligence/repomap/dead_code.go +++ b/internal/intelligence/repomap/dead_code.go @@ -1,3 +1,9 @@ +// dead_code.go flags top-level declarations (functions, +// methods, types, vars, consts) that appear to be unreferenced. It is +// deliberately conservative: declarations reachable from tests, interface +// implementations, or via reflection are not flagged. FormatDeadCode +// produces a human-readable summary, GenerateRemovalPlan a structured +// edit script. package repomap import ( diff --git a/internal/intelligence/repomap/depgraph.go b/internal/intelligence/repomap/depgraph.go index 952d5078..790dcf4d 100644 --- a/internal/intelligence/repomap/depgraph.go +++ b/internal/intelligence/repomap/depgraph.go @@ -1,3 +1,8 @@ +// depgraph.go constructs a package-level dependency graph +// for Go (via go.mod + go/parser ImportsOnly) and JavaScript/TypeScript +// (via package.json + import/require regexes). It computes topological +// order, layers, cycles, hot paths, and renders the result as DOT, +// Mermaid, or ASCII art for use in summaries and dashboards. package repomap import ( diff --git a/internal/intelligence/repomap/doc.go b/internal/intelligence/repomap/doc.go new file mode 100644 index 00000000..8613f3de --- /dev/null +++ b/internal/intelligence/repomap/doc.go @@ -0,0 +1,67 @@ +// Package repomap is the deep code-analysis engine that powers hawk's +// repository-mapping, symbol-extraction, and code-quality features. +// +// The package is built around the entry point Generate (in repomap.go), which +// walks a directory, dispatches each supported source file to a language-aware +// parser, and returns a RepoMap: a token-budgeted summary of files and their +// top-level symbols suitable for injection into LLM prompts. Around that core +// it accumulates a number of specialised analyses that all share the same +// parsed-symbol substrate: +// +// - Static analysis: call graph, import/dep graph, type hierarchy, +// interface extraction, code ownership, co-change statistics, and a +// Shapley-value ranker. +// - Search and navigation: BM25 semantic search, symbol-level PageRank, +// reranking, and a "go to definition / find references / find +// implementations" index that works without an LSP server. +// - Quality signals: cyclomatic complexity, code smells, repository health +// score, doc linter, dead-code detector, and a migration detector for +// deprecated language idioms. +// - API surface: a per-framework HTTP route scanner (Chi, net/http, Gin, +// Echo, Gorilla, Fiber) with OpenAPI export. +// - Incremental indexing: file-hash-keyed caches and an fsnotify-based +// watcher so that regeneration only re-processes files that changed. +// +// The package is intentionally stdlib-only at its core (go/parser, go/ast, +// go/token, encoding/*); the only third-party dependency is github.com/fsnotify +// /fsnotify for file watching, plus hawk's own internal/scoring and +// internal/ui/icons helpers where they are used. Tree-sitter is not required: +// Go is parsed with go/ast and other languages are handled by an enhanced +// regex extractor (see treesitter.go and parser_langs.go). +// +// # Dual-package relationship +// +// hawk ships a second package, internal/context/repomap, that exposes a much +// narrower surface - essentially just RepoMap(root, budget) (string, error). +// That package is the prompt-injection shim: it is what hawk's context layer +// calls when it needs a budgeted overview for the system prompt. It does its +// own AST parsing, PageRank pass, and rendering, and shares no code with this +// package beyond the name. The package you are reading is the deeper +// analysis engine that drives the higher-level navigators, quality tools, and +// search index, not the prompt-injection shim. See internal/context/repomap's +// own package comment for the shim side of the boundary. +// +// # Extension points +// +// - Add a new language parser by adding a new case to parseFileSymbols in +// repomap.go and a parseX function in parser_langs.go (or a new +// TreeSitterParser method in treesitter.go for languages that need +// scope-aware extraction). +// - Add a new code smell or quality heuristic by adding a Detector field +// on SmellDetector in smells.go and wiring it into ScanDirectory. +// - Add a new HTTP framework scanner by adding a ScanX function in +// api_scanner.go and a case in DetectFramework. +// +// # Performance and scaling +// +// Generate is O(N) in the number of files with a hard cap of Options.MaxFiles +// (default 500). The symbol cache (cache.go) is an in-process LRU keyed by +// (path, modtime) that is cleared on process exit; the IncrementalMap +// (incremental_map.go) provides a persistent on-disk cache at +// .hawk/repomap-cache.json keyed by SHA-256. Callers that need to operate on +// repositories with tens of thousands of files should set MaxFiles +// appropriately and prefer the incremental path. Static-analysis passes such +// as BuildCallGraph, BuildDepGraph, and the PageRank iteration in pagerank.go +// are O(V + E) per iteration over the symbol graph and scale linearly with +// the number of declarations, not the number of lines. +package repomap diff --git a/internal/intelligence/repomap/doc_test.go b/internal/intelligence/repomap/doc_test.go new file mode 100644 index 00000000..7bfb2521 --- /dev/null +++ b/internal/intelligence/repomap/doc_test.go @@ -0,0 +1,108 @@ +package repomap + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// ExampleGenerate is a worked example for the repomap package: it builds a +// tiny project under a temp directory, calls Generate, and renders the +// resulting RepoMap with Format. The output is verified to be non-empty +// and to contain the expected file and symbol names. +// +// This example is consumed by `go doc` and `godoc` to illustrate the +// package's primary entry point. +func ExampleGenerate() { + dir, err := os.MkdirTemp("", "repomap-example-") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + + // Two supported files: one Go, one Python. Both have a function + // and a type declaration so the example exercises the AST and + // regex paths. + if writeErr := os.WriteFile(filepath.Join(dir, "main.go"), []byte(`package main + +func main() {} + +type Server struct{} +`), 0o644); writeErr != nil { + panic(writeErr) + } + if writeErr := os.WriteFile(filepath.Join(dir, "app.py"), []byte(`class App: + pass + +def run(): + pass +`), 0o644); writeErr != nil { + panic(writeErr) + } + + rm, err := Generate(dir, Options{MaxFiles: 100, MaxTokens: 5000}) + if err != nil { + panic(err) + } + + out := rm.Format(5000) + if out == "" { + panic("expected non-empty output from Format") + } +} + +// TestExampleGenerate_isRunnable mirrors ExampleGenerate with assertions so +// `go test` validates the example end-to-end. The example function above +// has no // Output: comment (the output depends on the temp directory +// path, which is not stable across runs), so this test is the actual +// validator; godoc still picks up ExampleGenerate for the docs page. +func TestExampleGenerate_isRunnable(t *testing.T) { + dir := t.TempDir() + + if writeErr := os.WriteFile(filepath.Join(dir, "main.go"), []byte(`package main + +func main() {} + +type Server struct{} +`), 0o644); writeErr != nil { + t.Fatal(writeErr) + } + if writeErr := os.WriteFile(filepath.Join(dir, "app.py"), []byte(`class App: + pass + +def run(): + pass +`), 0o644); writeErr != nil { + t.Fatal(writeErr) + } + + rm, err := Generate(dir, Options{MaxFiles: 100, MaxTokens: 5000}) + if err != nil { + t.Fatalf("Generate failed: %v", err) + } + if rm == nil { + t.Fatal("expected non-nil RepoMap") + } + if len(rm.Files) != 2 { + t.Fatalf("expected 2 file maps, got %d", len(rm.Files)) + } + + out := rm.Format(5000) + if out == "" { + t.Fatal("expected non-empty formatted output") + } + if !strings.Contains(out, "main.go") { + t.Errorf("expected main.go in output, got:\n%s", out) + } + if !strings.Contains(out, "app.py") { + t.Errorf("expected app.py in output, got:\n%s", out) + } + // The Go file should have a func main and a type Server. + if !strings.Contains(out, "main") { + t.Errorf("expected main symbol in output, got:\n%s", out) + } + if !strings.Contains(out, "Server") { + t.Errorf("expected Server symbol in output, got:\n%s", out) + } +} diff --git a/internal/intelligence/repomap/doclint.go b/internal/intelligence/repomap/doclint.go index 9f30ae13..8296bdf1 100644 --- a/internal/intelligence/repomap/doclint.go +++ b/internal/intelligence/repomap/doclint.go @@ -1,3 +1,8 @@ +// doclint.go checks Go (and selectively other) source +// files for missing, outdated, incomplete, or unclear doc comments on +// exported symbols, and produces a per-file DocLintResult with a score +// and issue list. The result feeds the documentation-coverage dimension +// of the repository health score. package repomap import ( diff --git a/internal/intelligence/repomap/file_grouper.go b/internal/intelligence/repomap/file_grouper.go index b38d0ac0..5024432b 100644 --- a/internal/intelligence/repomap/file_grouper.go +++ b/internal/intelligence/repomap/file_grouper.go @@ -1,3 +1,8 @@ +// file_grouper.go identifies sets of files that should +// be edited as a unit (Go packages, feature slices, layered modules, +// test/source pairs, configuration groups). The resulting FileGroups are +// used by the context builder to include whole groups in a single +// prompt, not disjoint individual files. package repomap import ( diff --git a/internal/intelligence/repomap/gitignore.go b/internal/intelligence/repomap/gitignore.go index 0a098168..b241d72f 100644 --- a/internal/intelligence/repomap/gitignore.go +++ b/internal/intelligence/repomap/gitignore.go @@ -1,3 +1,7 @@ +// gitignore.go composes .gitignore rules from a project +// root, walking parent directories so that nested gitignore files (and +// the .git/info/exclude and global gitignore) all take effect. The resulting +// GitignoreRules is consulted by the file walk in Generate. package repomap import ( diff --git a/internal/intelligence/repomap/health_score.go b/internal/intelligence/repomap/health_score.go index 78e24dd7..1e2d35df 100644 --- a/internal/intelligence/repomap/health_score.go +++ b/internal/intelligence/repomap/health_score.go @@ -1,3 +1,8 @@ +// health_score.go rolls the individual signals +// (complexity, smells, dead code, doc coverage, ownership, duplication) +// into a single weighted HealthScore with per-dimension breakdowns, +// issue lists, and a letter grade. CompareScores produces a "before / +// after" diff suitable for commit messages or PR descriptions. package repomap import ( diff --git a/internal/intelligence/repomap/hierarchy.go b/internal/intelligence/repomap/hierarchy.go index 3f4f3f66..ebf60bed 100644 --- a/internal/intelligence/repomap/hierarchy.go +++ b/internal/intelligence/repomap/hierarchy.go @@ -1,3 +1,7 @@ +// hierarchy.go builds a three-level summary of a project +// (project -> package -> file) by grouping scanned files into Go packages +// and recording the exported symbols of each. It is consumed by the +// summary generator to produce an LLM-friendly project overview. package repomap import ( diff --git a/internal/intelligence/repomap/imports.go b/internal/intelligence/repomap/imports.go index 39f4be75..3568153e 100644 --- a/internal/intelligence/repomap/imports.go +++ b/internal/intelligence/repomap/imports.go @@ -1,3 +1,7 @@ +// imports.go builds a file-level ImportGraph from a Go +// project, mapping each file to its imports and reverse-mapping dependents. +// The graph is the cheapest cross-file signal in the package and is used by +// ChangeSetContext, PredictRelevantFiles, and the symbol reranker. package repomap import ( diff --git a/internal/intelligence/repomap/incremental.go b/internal/intelligence/repomap/incremental.go index 9d610c8e..598ffed4 100644 --- a/internal/intelligence/repomap/incremental.go +++ b/internal/intelligence/repomap/incremental.go @@ -1,3 +1,10 @@ +// incremental.go defines the CodeIndexer interface +// (the contract the repomap package uses to push code chunks to and +// query code chunks from a downstream store such as the memory package's +// YaadBridge) and IncrementalReindex, which diffs the project tree +// against a CodeIndexer's known file set and reindexes only added or +// changed files, removing deleted ones. ComputeFileHash returns the +// SHA-256 used to detect content changes. package repomap import ( diff --git a/internal/intelligence/repomap/incremental_map.go b/internal/intelligence/repomap/incremental_map.go index d1f85322..00c56a75 100644 --- a/internal/intelligence/repomap/incremental_map.go +++ b/internal/intelligence/repomap/incremental_map.go @@ -1,3 +1,8 @@ +// incremental_map.go is the persistent, on-disk symbol +// cache. It stores per-file SHA-256 hashes and the corresponding symbol +// lists in .hawk/repomap-cache.json (or a caller-provided directory). On +// regeneration only files whose hash changed are re-parsed, and deleted +// files are evicted, so the index stays accurate without a full rebuild. package repomap import ( diff --git a/internal/intelligence/repomap/interface_extract.go b/internal/intelligence/repomap/interface_extract.go index 73a4f7c6..1a3f966c 100644 --- a/internal/intelligence/repomap/interface_extract.go +++ b/internal/intelligence/repomap/interface_extract.go @@ -1,3 +1,8 @@ +// interface_extract.go returns just the exported API +// surface of a Go file (function signatures, type declarations, constants) +// with bodies stripped, for use in prompt contexts where body text is +// unaffordable. The output is roughly an order of magnitude smaller than +// the source it summarises. package repomap import ( diff --git a/internal/intelligence/repomap/migration_detector.go b/internal/intelligence/repomap/migration_detector.go index 648ccede..c93d2559 100644 --- a/internal/intelligence/repomap/migration_detector.go +++ b/internal/intelligence/repomap/migration_detector.go @@ -1,3 +1,8 @@ +// migration_detector.go applies a set of MigrationRule +// patterns to a project and reports MigrationOpportunities - patterns that +// could be updated to a newer API, a more idiomatic construct, or a more +// secure/performant alternative. AutoFix can rewrite selected patterns +// in-place. Results feed the health score's "deprecated APIs" dimension. package repomap import ( diff --git a/internal/intelligence/repomap/navigation.go b/internal/intelligence/repomap/navigation.go index ad8b9000..54102935 100644 --- a/internal/intelligence/repomap/navigation.go +++ b/internal/intelligence/repomap/navigation.go @@ -1,3 +1,7 @@ +// navigation.go implements an LSP-free "go to definition / +// find references / find implementations / find callers / find callees" +// index for a Go project. The index is built in-memory from go/parser ASTs +// and is consulted by the Hawk CLI's navigate subcommand. package repomap import ( diff --git a/internal/intelligence/repomap/ownership.go b/internal/intelligence/repomap/ownership.go index e4b78408..8af988e3 100644 --- a/internal/intelligence/repomap/ownership.go +++ b/internal/intelligence/repomap/ownership.go @@ -1,3 +1,7 @@ +// ownership.go combines git history with CODEOWNERS-style +// rules to compute per-file ownership: primary owner, contributors, +// recency, and bus-factor risk. It is consumed by the health-score +// dimension that flags orphaned files. package repomap import ( diff --git a/internal/intelligence/repomap/pagerank.go b/internal/intelligence/repomap/pagerank.go index 9502e5f1..cbb3153f 100644 --- a/internal/intelligence/repomap/pagerank.go +++ b/internal/intelligence/repomap/pagerank.go @@ -1,3 +1,9 @@ +// pagerank.go builds a per-symbol reference graph from a +// project (file:symbol nodes with directed "is referenced by" edges) and +// runs a damped PageRank iteration over it. BuildSymbolGraph accepts an +// optional IncrementalMap so only changed files are re-processed on each +// invocation. TopSymbols returns the highest-ranked symbols for prompt +// inclusion. package repomap import ( diff --git a/internal/intelligence/repomap/parser.go b/internal/intelligence/repomap/parser.go index 55260fd6..f69db5a6 100644 --- a/internal/intelligence/repomap/parser.go +++ b/internal/intelligence/repomap/parser.go @@ -1,3 +1,7 @@ +// parser.go contains the original regex-based Go symbol +// extractor. It is retained as a fast, dependency-free path; new code should +// prefer the AST-based EnhancedGoParser in parser_enhanced.go and the +// tree-sitter-style extractor in treesitter.go. package repomap import ( diff --git a/internal/intelligence/repomap/parser_enhanced.go b/internal/intelligence/repomap/parser_enhanced.go index e5d1e5f7..59b0f567 100644 --- a/internal/intelligence/repomap/parser_enhanced.go +++ b/internal/intelligence/repomap/parser_enhanced.go @@ -1,3 +1,7 @@ +// parser_enhanced.go wraps the standard-library go/parser +// and go/ast packages to extract symbols, methods, embedded types, and +// interface satisfaction from Go source. It replaces the legacy regex path +// for Go without requiring CGO or a tree-sitter binary. package repomap import ( diff --git a/internal/intelligence/repomap/parser_langs.go b/internal/intelligence/repomap/parser_langs.go index ed978255..3318b6e7 100644 --- a/internal/intelligence/repomap/parser_langs.go +++ b/internal/intelligence/repomap/parser_langs.go @@ -1,3 +1,8 @@ +// parser_langs.go holds the regex-based symbol extractors +// for every non-Go language the repomap knows about (C, C++, Java, C#, +// PHP, Ruby, Kotlin, Swift, Scala, Lua, Dart, Elixir, Haskell, ...). Each +// language has its own parseX function that the dispatcher in repomap.go +// selects by file extension. package repomap import ( diff --git a/internal/intelligence/repomap/patterns.go b/internal/intelligence/repomap/patterns.go index 328de272..9d1daec9 100644 --- a/internal/intelligence/repomap/patterns.go +++ b/internal/intelligence/repomap/patterns.go @@ -1,3 +1,7 @@ +// patterns.go loads a .hawk/repomap-patterns.json file +// (if present) describing which files to include or exclude from indexing, +// and falls back to DefaultIndexPatterns otherwise. The result is consumed +// by Generate, BuildSemanticIndex, and the watcher to skip unwanted trees. package repomap import ( diff --git a/internal/intelligence/repomap/predict.go b/internal/intelligence/repomap/predict.go index 2a4d0bac..88261a5c 100644 --- a/internal/intelligence/repomap/predict.go +++ b/internal/intelligence/repomap/predict.go @@ -1,3 +1,8 @@ +// predict.go predicts which files in a project are most +// likely to be relevant to the next edit, given the user's prompt, recent +// edits (a RecentEditTracker), the import graph, and a symbol map. The +// output is consumed by the change-set-aware context builder to populate +// a small, focused working set. package repomap import ( diff --git a/internal/intelligence/repomap/rerank.go b/internal/intelligence/repomap/rerank.go index c6747b71..57465183 100644 --- a/internal/intelligence/repomap/rerank.go +++ b/internal/intelligence/repomap/rerank.go @@ -1,3 +1,7 @@ +// rerank.go applies a BM25-over-BM25 reranking pass to a +// candidate set of code chunks. It is used after a faster but less precise +// retrieval step (semantic similarity, PageRank, or import-graph proximity) +// to surface the chunks that are most relevant to the query terms. package repomap import ( diff --git a/internal/intelligence/repomap/semantic.go b/internal/intelligence/repomap/semantic.go index a620a430..852bcbfc 100644 --- a/internal/intelligence/repomap/semantic.go +++ b/internal/intelligence/repomap/semantic.go @@ -1,3 +1,7 @@ +// semantic.go provides the lower-level chunked TF-IDF +// index used by the reranker. BuildSemanticIndex scans a directory, splits +// source files into ~40-line CodeChunks, and stores the result on disk +// (gob-encoded) for fast reload on subsequent sessions. package repomap import ( diff --git a/internal/intelligence/repomap/semantic_search.go b/internal/intelligence/repomap/semantic_search.go index 35aa23fc..f828163c 100644 --- a/internal/intelligence/repomap/semantic_search.go +++ b/internal/intelligence/repomap/semantic_search.go @@ -1,3 +1,8 @@ +// semantic_search.go is the BM25-ranked full-text search +// engine over the Document set produced by the navigation index. It +// tokenises queries, expands them with ExpandQuery, and returns SearchHit +// results with snippet extraction. This is the search backend used by the +// Hawk CLI's "find" subcommand. package repomap import ( diff --git a/internal/intelligence/repomap/shapley.go b/internal/intelligence/repomap/shapley.go index 6dbee1ff..de89da11 100644 --- a/internal/intelligence/repomap/shapley.go +++ b/internal/intelligence/repomap/shapley.go @@ -1,3 +1,7 @@ +// shapley.go applies a Shapley-value-based ranking over +// code chunks to score their marginal contribution to a task's context +// set. It complements PageRank by accounting for redundancy across the +// selected chunks, not just the in-graph centrality of each one. package repomap import ( diff --git a/internal/intelligence/repomap/smells.go b/internal/intelligence/repomap/smells.go index 4b13865c..dba5a95b 100644 --- a/internal/intelligence/repomap/smells.go +++ b/internal/intelligence/repomap/smells.go @@ -1,3 +1,10 @@ +// Package repomap: smells.go detects classic design smells (god object, +// long parameter list, feature envy, data clump, primitive obsession, +// long method, excessive imports, large file) via AST and heuristic rules. +// Each smell has a severity, a category, and a refactoring suggestion that +// is rendered by FormatSmells for inclusion in the health score. +// +// Package repomap package repomap import ( diff --git a/internal/intelligence/repomap/summary.go b/internal/intelligence/repomap/summary.go index 90837bb0..89653503 100644 --- a/internal/intelligence/repomap/summary.go +++ b/internal/intelligence/repomap/summary.go @@ -1,3 +1,8 @@ +// summary.go produces the LLM-facing CodebaseSummary: +// a high-level description of the project, its packages, entry points, +// key files, and inferred architecture. RenderForPrompt and RenderCompact +// emit the summary in token-bounded forms suitable for injection at the +// start of a long session. package repomap import ( diff --git a/internal/intelligence/repomap/treesitter.go b/internal/intelligence/repomap/treesitter.go index 00818536..c6f2cdd0 100644 --- a/internal/intelligence/repomap/treesitter.go +++ b/internal/intelligence/repomap/treesitter.go @@ -1,3 +1,8 @@ +// treesitter.go is the "tree-sitter-inspired" scope-aware +// extractor. It uses go/ast for Go and a small scope-tracking regex engine +// for other languages, returning the same shape of symbol list a real +// tree-sitter parser would. ParseFileEnhanced and ParseSourceEnhanced are +// the canonical entry points used by the rest of the package. package repomap import ( diff --git a/internal/intelligence/repomap/watcher.go b/internal/intelligence/repomap/watcher.go index 6baf900a..46321520 100644 --- a/internal/intelligence/repomap/watcher.go +++ b/internal/intelligence/repomap/watcher.go @@ -1,3 +1,7 @@ +// watcher.go wraps fsnotify.Watcher to monitor a project +// tree for file-system events and invoke a callback when a supported +// source file changes. It is the runtime hook used to keep the in-memory +// symbol index in sync with the working tree during a long-running session. package repomap import (