feat(consolidation): cache-segment planner; curator+toolset CLI; docs

README.md

@@ -260,6 +260,27 @@ Define personas and eval tasks in YAML (in addition to markdown personas), inclu
 
 An optional, non-excludable org-policy rule tier (highest precedence) with HTML-comment stripping of rule files for IT-managed deployments.
 
+### Adopted Capabilities (env-gated)
+
+Features adopted from open-source agent projects. All are off by default unless explicitly enabled; see each section for details.
+
+| Feature | Flag / Command | What it does |
+|---|---|---|
+| Best-of-N fan-out | `hawk exec --fanout N` | Run the same prompt in N isolated worktrees, compare, merge winner |
+| Completion notifications | `HAWK_NOTIFY_WEBHOOK_URL` / `HAWK_NOTIFY_TELEGRAM_TOKEN` + `_CHAT_ID` | Webhook or Telegram ping when a run finishes |
+| Incremental system-context | `HAWK_INCREMENTAL_CONTEXT=1` | Reconcile dynamic sections instead of rebuilding the prompt |
+| Tool-catalog shrink | `HAWK_TOOL_SHRINK=1` | Compress the tool catalog sent on every request |
+| Compaction segments | `HAWK_COMPACTION_SEGMENT_DETAIL=verbose\|balanced\|minimal\|none` | Persist verbatim compacted turns to disk |
+| Skill curator | `hawk skills curator status/run/pin/unpin/archive` + `HAWK_SKILL_CURATOR=1` | Auto-archive cold agent-created skills (recoverable) |
+| Structural code match | `CodeMatch` tool | Tree-sitter query search over Go/Python/TS/TSX |
+| Composable toolsets | `hawk toolset [name]` + `Toolset` tool | Named tool groups (research, dev, ops, full_stack) |
+| App verification | `AppVerify` tool | Boot-smoke check with readiness polling and evidence artifacts |
+| Media generation | `GenerateMedia` tool (needs a wired engine) | Image/video generation with local persistence |
+| Git-tree file snapshots | `internal/gitsnapshot` | Content-addressed tree capture/diff/preview/restore |
+| Turn-boundary rewind | `internal/filestate` | Per-prompt before/after snapshots with durable store |
+| Path reservations | `internal/multiagent` ledger | Detect overlapping-file changes between parallel branches |
+| Live agent status | `GET /v1/agent/status` (daemon) | Machine-readable working/idle/stale per session |
+
 ## Usage
 
 ### Interactive Mode

cmd/skills_curator_cmd.go

@@ -0,0 +1,98 @@
+package cmd
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/GrayCodeAI/hawk/internal/intelligence/skillcurator"
+	"github.com/GrayCodeAI/hawk/internal/storage"
+	"github.com/spf13/cobra"
+)
+
+// skillsCuratorCmd exposes the background skill curator (adopted from Hermes
+// Agent) as a CLI surface: review/archive/pin/unpin over agent-created skills
+// in ~/.hawk/skills.
+var skillsCuratorCmd = &cobra.Command{
+	Use:   "curator [command]",
+	Short: "Skill lifecycle curation (status, run, pin, unpin, archive)",
+	Long: `Maintain the agent-created skill collection.
+
+  status            List skills with lifecycle status and usage
+  run               Run the inactivity review now (archives cold skills)
+  pin <name>        Pin a skill (auto-transitions skip pinned skills)
+  unpin <name>      Remove a pin
+  archive <name>    Move a skill to .archive/ (recoverable, never deleted)
+
+The review is inactivity-triggered: only agent-created skills that have been
+used before and have gone cold are archived; installed third-party skills,
+never-used skills, and pinned skills are left alone.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if len(args) == 0 {
+			return cmd.Help()
+		}
+		c, err := newCurator()
+		if err != nil {
+			return err
+		}
+		switch args[0] {
+		case "status", "list":
+			skills, err := c.List()
+			if err != nil {
+				return err
+			}
+			if len(skills) == 0 {
+				fmt.Println("No curated skills found.")
+				return nil
+			}
+			for _, s := range skills {
+				last := "-"
+				if !s.LastUsed.IsZero() {
+					last = s.LastUsed.Format("2006-01-02")
+				}
+				fmt.Printf("%-24s %-9s uses=%-4d last=%s\n", s.Name, s.Status, s.UseCount, last)
+			}
+			return nil
+		case "run":
+			archived, err := c.ForceReview()
+			if err != nil {
+				return err
+			}
+			if len(archived) == 0 {
+				fmt.Println("Review complete: nothing to archive.")
+				return nil
+			}
+			fmt.Printf("Archived %d cold skill(s):\n", len(archived))
+			for _, n := range archived {
+				fmt.Printf("  - %s (recoverable from .archive/)\n", n)
+			}
+			return nil
+		case "pin":
+			return requireArg(args, func(name string) error { return c.Pin(name) })
+		case "unpin":
+			return requireArg(args, func(name string) error { return c.Unpin(name) })
+		case "archive":
+			return requireArg(args, func(name string) error { return c.Archive(name, "archived via CLI") })
+		default:
+			return fmt.Errorf("unknown curator command %q (use status, run, pin, unpin, archive)", args[0])
+		}
+	},
+}
+
+func requireArg(args []string, fn func(string) error) error {
+	if len(args) < 2 {
+		return fmt.Errorf("%s requires a skill name", args[0])
+	}
+	return fn(args[1])
+}
+
+func newCurator() (*skillcurator.Curator, error) {
+	dir := filepath.Join(storage.StateDir(), "skills")
+	return skillcurator.New(skillcurator.Config{
+		SkillsDir: dir,
+		StateFile: filepath.Join(dir, ".curator_state.json"),
+	})
+}
+
+func init() {
+	skillsCmd.AddCommand(skillsCuratorCmd)
+}

cmd/toolset_cmd.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/GrayCodeAI/hawk/internal/toolset"
+	"github.com/spf13/cobra"
+)
+
+// toolsetCmd exposes named, composable tool groups (adopted from Hermes
+// Agent's toolset system) as a CLI surface: list the available groups or
+// resolve one to its concrete, de-duplicated, sorted tool list.
+var toolsetCmd = &cobra.Command{
+	Use:   "toolset [name]",
+	Short: "List or resolve composable tool groups",
+	Long: `Named, composable tool groups for scoping an agent's tool surface.
+
+  hawk toolset            List available toolsets
+  hawk toolset research   Resolve 'research' to its concrete tool list
+
+Toolsets compose from other toolsets; resolving expands Requires
+transitively (cycle-safe) and de-duplicates.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		reg, err := toolset.NewRegistry(toolset.Defaults())
+		if err != nil {
+			return err
+		}
+		if len(args) == 0 {
+			fmt.Println("Available toolsets: " + strings.Join(reg.Names(), ", "))
+			return nil
+		}
+		name := args[0]
+		tools, err := reg.Resolve(name)
+		if err != nil {
+			return err
+		}
+		payload := map[string]interface{}{"toolset": name, "tools": tools, "count": len(tools)}
+		out, _ := json.MarshalIndent(payload, "", "  ")
+		fmt.Println(string(out))
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(toolsetCmd)
+}

internal/engine/cache_gate.go

@@ -1,36 +1,9 @@
 package engine
 
-import (
-	"encoding/json"
-	"strings"
-
-	"github.com/GrayCodeAI/hawk/internal/types"
-)
-
-// Prompt-cache break-even gate, adopting caveman's cacheengine arithmetic in
-// miniature: provider-native caching charges a write premium on cached input
-// (Anthropic 5m: write=1.25x, read=0.1x) and pays off only when the stable
-// prefix is reused. Below the break-even prefix size the premium costs more
-// than one reuse saves, so caching stays OFF rather than burning the write.
-//
-// Full segment planning and key-sharding belong in eyrie; this is the
-// client-side gate only.
-
-// cacheMinPrefixBytes is the smallest stable prefix worth a cache write.
-// ~8 KiB approximates 2k tokens: at Anthropic economics, two reuses of a
-// 2k-token prefix already beat paying full price twice (2x1.0 > 1.25+0.1).
-const cacheMinPrefixBytes = 8 * 1024
-
-// cacheDecision reports whether to request provider-native prompt caching
-// for this call. Deterministic and pure so it can be tested without a
-// provider connection.
-func cacheDecision(provider, systemPrompt string, tools []types.EyrieTool) bool {
-	if !strings.EqualFold(provider, "anthropic") {
-		return false // other providers: implicit caching; no explicit controls
-	}
-	stable := len(systemPrompt)
-	if raw, err := json.Marshal(tools); err == nil {
-		stable += len(raw)
-	}
-	return stable >= cacheMinPrefixBytes
-}
+// Prompt-cache break-even gate: provider-native caching charges a write
+// premium on cached input (Anthropic 5m: write=1.25x, read=0.1x) and pays off
+// only when the stable prefix is reused. The deterministic client planner in
+// cache_planner.go (planCache / cacheDecision) implements this arithmetic:
+// segments the stable prefix, computes breakpoints, and enables caching only
+// when the expected reuse count beats the write premium. Full wire-format
+// lowering and fleet-wide key-sharding remain eyrie-side.

internal/engine/cache_planner.go

@@ -0,0 +1,117 @@
+package engine
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/GrayCodeAI/hawk/internal/types"
+)
+
+// Prompt-cache segment planning, extending the caveman-style break-even gate
+// into a full client-side planner: the stable prefix (system prompt + tool
+// catalog) is split into cacheable segments, breakpoints are computed at the
+// last tool and last system boundaries, and the whole thing is enabled only
+// when the provider economics pay off for the expected reuse count. Full
+// wire-format lowering and key sharding across a fleet remain eyrie-side;
+// this is the deterministic client planner.
+
+// Anthropic 5m cache economics: uncached = 1.0x, cache write = 1.25x,
+// cache read = 0.1x per reuse. The write premium (0.25x) is paid once.
+const (
+	cacheWritePremium = 0.25
+	cacheReadCost     = 0.1 // per-reuse cost of reading a cached segment
+	// cacheMinSegmentBytes: segments below this are not worth a cache write.
+	cacheMinSegmentBytes = 8 * 1024
+	// cacheDefaultReuse is the expected number of turns reusing the prefix.
+	cacheDefaultReuse = 2
+	// cacheMinPrefixBytes is the historical name for the minimum stable-prefix
+	// size worth a cache write (alias for cacheMinSegmentBytes).
+	cacheMinPrefixBytes = cacheMinSegmentBytes
+)
+
+// CacheSegment is one cacheable unit of the stable prefix.
+type CacheSegment struct {
+	Index int    `json:"index"`
+	Label string `json:"label"` // "system" | "tools"
+	Bytes int    `json:"bytes"`
+}
+
+// CachePlan is the outcome of cache planning for one call.
+type CachePlan struct {
+	Enabled      bool           `json:"enabled"`
+	Provider     string         `json:"provider"`
+	Segments     []CacheSegment `json:"segments"`
+	Breakpoints  int            `json:"breakpoints"` // number of cache breakpoints to emit
+	ReuseCount   int            `json:"reuse_count"`
+	WriteBytes   int            `json:"write_bytes"`
+	ReadSaving   int            `json:"read_saving_bytes"`
+	UncachedCost int            `json:"uncached_cost_bytes"`
+	CachedCost   int            `json:"cached_cost_bytes"`
+	Reason       string         `json:"reason,omitempty"`
+}
+
+// planCache computes whether and how to cache the stable prefix for a call.
+// Deterministic and pure so it can be tested without a provider connection.
+func planCache(provider, systemPrompt string, tools []types.EyrieTool, expectedReuse int) CachePlan {
+	if expectedReuse <= 0 {
+		expectedReuse = cacheDefaultReuse
+	}
+	if !strings.EqualFold(provider, "anthropic") {
+		return CachePlan{Provider: provider, ReuseCount: expectedReuse, Reason: "no explicit cache controls for provider"}
+	}
+
+	plan := CachePlan{Provider: "anthropic", ReuseCount: expectedReuse}
+	total := 0
+	if systemPrompt != "" {
+		plan.Segments = append(plan.Segments, CacheSegment{Index: 0, Label: "system", Bytes: len(systemPrompt)})
+		total += len(systemPrompt)
+	}
+	if len(tools) > 0 {
+		if raw, err := json.Marshal(tools); err == nil && len(raw) > 0 {
+			plan.Segments = append(plan.Segments, CacheSegment{Index: len(plan.Segments), Label: "tools", Bytes: len(raw)})
+			total += len(raw)
+		}
+	}
+	plan.WriteBytes = total
+
+	if len(plan.Segments) == 0 {
+		plan.Reason = "no stable prefix to cache"
+		return plan
+	}
+
+	// A tiny prefix is not worth the write premium (matches the old gate).
+	if total < cacheMinSegmentBytes {
+		plan.Reason = "prefix below break-even size"
+		return plan
+	}
+
+	// Breakpoints: one at the last system boundary and one at the last tool
+	// boundary. We emit breakpoints on every segment so a later prefix reuse
+	// hits an early breakpoint (Anthropic caches at the nearest breakpoint
+	// before reused content). 2 segments -> 2 breakpoints.
+	plan.Breakpoints = len(plan.Segments)
+
+	// Economics: break-even reuse R satisfies
+	//   R*S > 1.25*S + (R-1)*0.1*S  =>  R > 1.25 + 0.1*(R-1)
+	//   => 0.9*R > 1.15 => R > 1.278
+	// So caching pays off for reuse >= 2. Compute costs for the given reuse.
+	plan.UncachedCost = total * expectedReuse
+	write := int(float64(total) * (1 + cacheWritePremium))
+	reads := int(float64(total) * cacheReadCost * float64(expectedReuse-1))
+	plan.CachedCost = write + reads
+	plan.ReadSaving = plan.UncachedCost - plan.CachedCost
+
+	if plan.ReadSaving <= 0 {
+		plan.Reason = "caching does not beat uncached at expected reuse"
+		return plan
+	}
+	plan.Enabled = true
+	plan.Reason = "break-even satisfied"
+	return plan
+}
+
+// cacheDecision reports whether to request provider-native caching, delegating
+// to the planner at the default reuse. Kept for backward compatibility.
+func cacheDecision(provider, systemPrompt string, tools []types.EyrieTool) bool {
+	return planCache(provider, systemPrompt, tools, cacheDefaultReuse).Enabled
+}