Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions agents/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ its own extra (`[bugzilla]`, `[firefox]`):
```python
from agent_tools import bugzilla
from agent_tools.claude_sdk import build_sdk_server
server = build_sdk_server("bugzilla", BugzillaContext(client=...), bugzilla.TOOLS)
```

Expand Down
12 changes: 12 additions & 0 deletions agents/build-repair/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,18 @@ Second stage - fixing:

The result reports `blamed_commit` so the caller can attribute the failure to a developer.

## Email notification

A run that produced a patch records an `email.send` action carrying the analysis, the
blamed commit and the patch, addressed to that commit's author and to the developer who
pushed the failing change (the hackbot team is copied apply-side). A run that proposed no
patch is a transient or not-to-blame failure and is not emailed -- see
`NOTIFY_ONLY_WITH_PATCH` in [config.py](hackbot_agents/build_repair/config.py).

The email is delivered by the apply step, not from the run, so it is visible in the
hackbot UI before it lands and is delivered at most once. `build-repair` opts into
auto-apply, so a succeeded run reports without waiting for a human.

## Test the agent

```sh
Expand Down
46 changes: 43 additions & 3 deletions agents/build-repair/hackbot_agents/build_repair/__main__.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
import logging

from hackbot_runtime import HackbotContext, run_async
from hackbot_runtime.actions.email import record_email
from pydantic_settings import BaseSettings, SettingsConfigDict

from .agent import BuildRepairResult, run_build_repair
from .resolve import resolve_push
from .config import NOTIFY_ONLY_WITH_PATCH
from .notify import build_email, recipients, resolve_author_email
from .resolve import PushInfo, resolve_push

logger = logging.getLogger(__name__)


class AgentInputs(BaseSettings):
Expand Down Expand Up @@ -37,9 +44,9 @@ async def main(ctx: HackbotContext) -> BuildRepairResult:

# Pin the checkout to the failure commit and fetch deep enough to include the
# whole push, so the agent can `git show` every commit in it.
await ctx.prepare_repo(ref=git_commits[0], depth=len(git_commits) + 1)
await ctx.prepare_repo(ref=push.git_commits[0], depth=len(push.git_commits) + 1)

return await run_build_repair(
result = await run_build_repair(
bugzilla_mcp_server={
"type": "http",
"url": inputs.bugzilla_mcp_url,
Expand All @@ -59,6 +66,39 @@ async def main(ctx: HackbotContext) -> BuildRepairResult:
publish_file=ctx.publish_file,
)

try:
_record_analysis_email(ctx, result, push, task_id)
except Exception:
# A notification is never worth losing a finished analysis over.
logger.exception("Could not record the failure-analysis email")
return result


def _record_analysis_email(
ctx: HackbotContext, result: BuildRepairResult, push: PushInfo, task_id: str
) -> None:
has_patch = ctx.source_changed
if NOTIFY_ONLY_WITH_PATCH and not has_patch:
logger.info("Run produced no patch; not emailing the failure analysis")
return

blamed_author = resolve_author_email(ctx.repo_path, result.blamed_commit)
subject, body = build_email(
result,
push,
task_id=task_id,
run_id=ctx.run_id,
has_patch=has_patch,
blamed_author=blamed_author,
)
record_email(
ctx.actions,
to=recipients(push, blamed_author),
subject=subject,
body_markdown=body,
attach_patch=has_patch,
)


if __name__ == "__main__":
run_async(main)
4 changes: 4 additions & 0 deletions agents/build-repair/hackbot_agents/build_repair/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
ANALYSIS_MODEL = "claude-opus-4-8"
FIX_MODEL = "claude-opus-4-8"

# A run that proposed no patch is a transient or not-to-blame failure; emailing the
# developer about it is noise.
NOTIFY_ONLY_WITH_PATCH = True

# Bugzilla MCP tool names as exposed to the agent (mcp__<server>__<tool>).
BUGZILLA_READ_TOOLS = [
"mcp__bugzilla__search_bugs",
Expand Down
166 changes: 166 additions & 0 deletions agents/build-repair/hackbot_agents/build_repair/notify.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

"""The email a finished run sends about the build failure.

Recorded as an ``email.send`` action rather than sent from the run, so it is
visible in the hackbot UI before it lands and is delivered at most once (see
``hackbot_runtime.actions.email``).

It reaches the developer who pushed the failing change and the author the agent
blamed; the team address is added apply-side. Every identifier a recipient would
otherwise have to look up -- revisions, task, bug, run -- is a link.
"""

from __future__ import annotations

import subprocess
from pathlib import Path

from hackbot_runtime.actions.email import PATCH_PLACEHOLDER, demote_headings
from hackbot_runtime.actions.slack import HACKBOT_UI_URL

from .agent import BuildRepairResult
from .resolve import PushInfo

GIT_COMMIT_URL = "https://github.com/mozilla-firefox/firefox/commit/{sha}"
HG_REV_URL = "https://hg.mozilla.org/mozilla-unified/rev/{rev}"
TASK_URL = "https://firefox-ci-tc.services.mozilla.com/tasks/{task_id}"
TREEHERDER_JOB_URL = (
"https://treeherder.mozilla.org/#/jobs"
"?repo={project}&revision={revision}&selectedTaskRun={task_id}"
)
BUG_URL = "https://bugzilla.mozilla.org/show_bug.cgi?id={bug_id}"
RUN_URL = HACKBOT_UI_URL.rstrip("/") + "/runs/{run_id}"


def resolve_author_email(source_repo: Path, sha: str | None) -> str | None:
"""The blamed commit's author email, so the notification reaches them."""
if not sha:
return None
try:
proc = subprocess.run(
["git", "-C", str(source_repo), "show", "-s", "--format=%ae", sha],
capture_output=True,
text=True,
)
except OSError:
return None
return proc.stdout.strip() or None


def _link(url: str, label: str) -> str:
return f"[{label}]({url})"


def recipients(push: PushInfo, blamed_author: str | None) -> list[str]:
"""Who the failure concerns: the blamed author first, then the pusher."""
return [address for address in (blamed_author, push.developer_email) if address]


def _why_section(
push: PushInfo, blamed_commit: str | None, author: str | None
) -> list[str]:
"""Explain why each recipient is on the email."""
notes = []
if push.developer_email:
notes.append(
f"- **{push.developer_email}** pushed the change whose build failed."
)
if blamed_commit:
who = f"**{author}** authored" if author else "The agent believes"
link = _link(
GIT_COMMIT_URL.format(sha=blamed_commit), f"`{blamed_commit[:12]}`"
)
notes.append(f"- {who} {link}, which introduced the failure.")
return ["", "## Why you're receiving this", "", *notes] if notes else []


def _analysis_sections(result: BuildRepairResult) -> list[str]:
lines: list[str] = []
for text, title in ((result.summary, "Summary"), (result.analysis, "Analysis")):
if text:
lines += ["", f"## {title}", "", demote_headings(text)]
return lines


def build_email(
result: BuildRepairResult,
push: PushInfo,
*,
task_id: str,
run_id: str,
has_patch: bool = False,
blamed_author: str | None = None,
) -> tuple[str, str]:
"""The subject and markdown body of the build-failure email."""
failure_commit = push.git_commits[0]
subject = (
f"[build-repair] Build failure analysis for "
f"{push.project}@{failure_commit[:12]}"
)

lines = [
"# Build failure analysis",
"",
f"- **Repository:** {push.project}",
"- **Revision (git):** "
+ _link(GIT_COMMIT_URL.format(sha=failure_commit), f"`{failure_commit[:12]}`"),
"- **Failed task:** " + _link(TASK_URL.format(task_id=task_id), f"`{task_id}`"),
]
# Absent only when the run was pinned to a git commit by hand; both lines are
# keyed on the hg revision, so they go together.
if push.hg_revision:
lines += [
"- **Revision (hg):** "
+ _link(
HG_REV_URL.format(rev=push.hg_revision), f"`{push.hg_revision[:12]}`"
),
"- **Treeherder:** "
+ _link(
TREEHERDER_JOB_URL.format(
project=push.project, revision=push.hg_revision, task_id=task_id
),
"jobs",
),
]

if result.blamed_commit:
by = f" by {blamed_author}" if blamed_author else ""
lines.append(
"- **Likely culprit:** "
+ _link(
GIT_COMMIT_URL.format(sha=result.blamed_commit),
f"`{result.blamed_commit[:12]}`",
)
+ by
)
else:
lines.append(
"- **Not caused by this push:** the failure is pre-existing or "
"infrastructure, so no commit here is blamed."
)
if result.bug_id:
lines.append(
"- **Bug:** "
+ _link(BUG_URL.format(bug_id=result.bug_id), str(result.bug_id))
)
lines.append("- **Run details:** " + RUN_URL.format(run_id=run_id))

lines += _why_section(push, result.blamed_commit, blamed_author)
lines += _analysis_sections(result)

if result.local_build_verified is not None:
lines += [
"",
"## Verification",
"",
f"- Local build verified: {result.local_build_verified}",
]
if has_patch:
# The diff itself is substituted for the placeholder when the mail is sent,
# from the same artifact it attaches.
lines += ["", "## Proposed patch", "", "```diff", PATCH_PLACEHOLDER, "```"]
return subject, "\n".join(lines)
29 changes: 20 additions & 9 deletions agents/build-repair/hackbot_agents/build_repair/resolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.

"""Resolve a Taskcluster build-failure task into the commits to repair.
"""Resolve a Taskcluster build-failure task into the push to repair.

Given a failing build task id, look up its push: the failure (head) commit the
tree is checked out at, plus the other commits that landed in the same push so
the agent can blame the one that broke the build. Uses the same public
Taskcluster / lando / pushlog lookups the pulse listener does, so the agent
derives the commits itself from a task id.
derives everything it reports from a task id.
"""

from __future__ import annotations
Expand Down Expand Up @@ -43,6 +43,18 @@ def _get_json(url: str) -> dict:
return resp.json()


def _task(task_id: str) -> dict:
return _get_json(_TC_TASK_URL.format(task_id=task_id))


def _task_push(task: dict) -> tuple[str | None, str | None]:
tags = task.get("tags") or {}
return (
tags.get("project"),
(task.get("payload") or {}).get("env", {}).get("GECKO_HEAD_REV"),
)


def _hg_to_git(rev: str) -> str:
return _get_json(_LANDO_HG2GIT.format(rev=rev))["git_hash"]

Expand Down Expand Up @@ -83,16 +95,13 @@ class PushInfo:
project: str | None
hg_revision: str | None
git_commits: list[str]
# ``createdForUser``: who pushed the change that failed to build.
developer_email: str | None = None


def task_push(task_id: str) -> tuple[str | None, str | None]:
"""The ``(project, hg_revision)`` a task ran on; what Treeherder is keyed on."""
task = _get_json(_TC_TASK_URL.format(task_id=task_id))
tags = task.get("tags") or {}
return (
tags.get("project"),
(task.get("payload") or {}).get("env", {}).get("GECKO_HEAD_REV"),
)
return _task_push(_task(task_id))


def resolve_push(task_id: str, git_commit: str | None = None) -> PushInfo:
Expand All @@ -102,7 +111,8 @@ def resolve_push(task_id: str, git_commit: str | None = None) -> PushInfo:
task is still fetched for its revision. Raises on network errors or when the
failure commit cannot be determined.
"""
project, hg_rev = task_push(task_id)
task = _task(task_id)
project, hg_rev = _task_push(task)

push = _push_git_commits(project, hg_rev) if hg_rev and project else []

Expand All @@ -118,4 +128,5 @@ def resolve_push(task_id: str, git_commit: str | None = None) -> PushInfo:
project=project,
hg_revision=hg_rev,
git_commits=[failure_commit] + [c for c in push if c != failure_commit],
developer_email=(task.get("tags") or {}).get("createdForUser"),
)
3 changes: 3 additions & 0 deletions agents/build-repair/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,6 @@ build-backend = "hatchling.build"

[tool.hatch.build.targets.wheel]
packages = ["hackbot_agents", "evals"]

[tool.pytest.ini_options]
testpaths = ["tests"]
Empty file.
Loading