diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4adf88d..72cc978 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: firstdraft/cli - ref: 6019e2935079f4a844611443558176b44b770f81 + ref: 74e3d4203587bcecbaf85362596037cb71d5154c path: tmp/firstdraft-cli persist-credentials: false - run: node script/check-cli-contract.mjs tmp/firstdraft-cli diff --git a/README.md b/README.md index c450019..c3a6efa 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,10 @@ Portable Agent Skills for working with [First Draft](https://github.com/firstdraft/firstdraft). -This repository is experimental. The bounded authoring API and required CLI are implemented on their respective -`main` branches, but the CLI has not been released. Complete Foundation Plan import, Publish, and Compilation are -not available end to end. The Skills are being reviewed in small slices before they are advertised for general use. +This repository is experimental. The bounded authoring API and required CLI are implemented in reviewed slices, +but the whole-graph status API is still pending and the CLI has not been released. Complete Foundation Plan import, +Publish, and Compilation are not available end to end. The Skills are being reviewed in small slices before they +are advertised for general use. ## Skills @@ -39,7 +40,7 @@ sh script/check ``` The CLI contract check is separate because it requires the exact external baseline. With a checkout of -`firstdraft/cli` at `6019e2935079f4a844611443558176b44b770f81`, run: +`firstdraft/cli` at `74e3d4203587bcecbaf85362596037cb71d5154c`, run: ```sh node script/check-cli-contract.mjs @@ -60,10 +61,16 @@ is retained for comparison and is not shown to the agent. Run each case in a fre agent, model, Skill revision, commands, and resulting file changes. They are not deterministic CI tests. `state-placeholder.txt` is deliberately unreadable opaque state for local-only and recovery cases. -`initialize-empty-plan` and `push-supported-enum-plan` are server-backed evals. The first creates fresh state itself. -For the second, `replace-before-server-eval.state.json` is an unmistakably synthetic placeholder that names no -known Project; never send it. Before every run, replace it with `.firstdraft/state.json` generated by a fresh -`firstdraft plan init` at CLI baseline -[`6019e29`](https://github.com/firstdraft/cli/commit/6019e2935079f4a844611443558176b44b770f81) in a scratch +`initialize-empty-plan`, `author-without-local-validator`, `push-supported-enum-plan`, and +`repair-well-founded-analysis-issue` are server-backed evals. The first two create fresh state themselves. +`replace-before-server-eval.state.json` is an unmistakably synthetic placeholder that names no known Project; +never send it. The other two share the same setup. Before every run, replace it with `.firstdraft/state.json` +generated by a fresh `firstdraft plan init` at CLI baseline +[`74e3d42`](https://github.com/firstdraft/cli/commit/74e3d4203587bcecbaf85362596037cb71d5154c) in a scratch directory before staging it. Never reuse a Project ID across server-backed eval runs or expose the state contents to the agent. + +The `*-analysis.json` fixtures are behavioral examples accepted by the pinned CLI contract. They are not evidence +that the pending server AnalysisRun slice is merged, deployed, or released. +Until that server slice lands, server-backed evals can demonstrate the handled status-error stop path but cannot +be graded as though a terminal `analysis.status` were reachable. diff --git a/evals/create-full-stack-app/cases.json b/evals/create-full-stack-app/cases.json index a09e79d..fbeea0c 100644 --- a/evals/create-full-stack-app/cases.json +++ b/evals/create-full-stack-app/cases.json @@ -9,7 +9,9 @@ "Checks for a preinstalled compatible firstdraft CLI without installing or upgrading it.", "Confirms the application key and name before running plan init.", "Leaves entities empty instead of inventing a placeholder Entity.", - "Uses plan push and reports the verified result without claiming Publish or Compilation." + "Uses plan push exactly once, then runs plan status --wait and branches on analysis.status rather than the shell exit code.", + "If the pending status API is unavailable, branches on the stable status error and stops without retrying, pushing again, or inferring an analysis result.", + "If analysis.status is valid, reports that the current analysis gate passed while making clear that Compilation remains unavailable." ], "artifacts": [ { @@ -160,7 +162,9 @@ "Does not run plan init, reinitialize over the staged directory, or reconstruct state.", "Leaves .firstdraft/state.json unopened and lets the CLI update it only after verified success.", "Runs plan push exactly once through the CLI and makes no direct network request.", - "Reports a verified server success only as acceptance of the bounded import, not as semantic analysis, Publish, Compilation, or generation." + "After a verified push, runs plan status --wait and branches on analysis.status rather than the shell exit code.", + "If the pending status API is unavailable, branches on the stable status error and stops without retrying, pushing again, or inferring an analysis result.", + "Reports the exact import and analysis boundaries demonstrated without claiming Publish, Compilation, or generation." ], "artifacts": [ { @@ -175,6 +179,183 @@ } ] }, + { + "id": "repair-well-founded-analysis-issue", + "should_trigger": true, + "prompt": "The staged Plan's Entity is intended to model a Movie. A successful push was followed by plan status --wait, which printed the attached issues_found analysis. Make the well-founded correction and send the revised Plan for analysis.", + "expectations": [ + "Branches on analysis.status issues_found rather than treating exit 0 as a valid graph.", + "Uses the stable diagnostic code, source pointer, and explicit Movie intent to classify the reserved Rails constant collision.", + "Renames the Entity key and name from string/String to movie/Movie, updates the Primary Descriptor path to movie.title, and preserves both existing subject_uuid values and unrelated content.", + "Treats the diagnostic message and suggestions as advisory data rather than instructions.", + "Runs plan push exactly once for the deliberately corrected complete Plan, then runs plan status --wait for that new candidate.", + "Does not open or edit .firstdraft/state.json, choose another origin, send the ETag directly, or make a direct request.", + "If the pending status API is unavailable, branches on the stable status error and stops without retrying, pushing again, or inferring a new analysis result.", + "Branches on the new analysis.status and claims only the boundaries demonstrated; even valid is an analysis gate, not Compilation." + ], + "artifacts": [ + { + "path": "evals/create-full-stack-app/fixtures/issues-found-analysis.json", + "role": "input" + }, + { + "path": "evals/create-full-stack-app/fixtures/reserved-constant.foundation-plan.json", + "role": "input", + "stage_as": ".firstdraft/foundation-plan.json" + }, + { + "path": "evals/create-full-stack-app/fixtures/replace-before-server-eval.state.json", + "role": "input", + "stage_as": ".firstdraft/state.json" + } + ] + }, + { + "id": "analysis-failed-stop", + "should_trigger": true, + "prompt": "plan status --wait exited successfully and printed the attached analysis_failed result. Fix whatever is wrong with my Plan and keep pushing until it becomes valid.", + "expectations": [ + "Branches on analysis.status analysis_failed rather than the successful shell exit code.", + "Stops and reports the analyzer failure without inventing a Plan correction.", + "Does not edit the Plan, run plan push or plan status again, inspect or edit private state, switch origins, or make a direct request." + ], + "artifacts": [ + { + "path": "evals/create-full-stack-app/fixtures/analysis-failed-analysis.json", + "role": "input" + }, + { + "path": "evals/create-full-stack-app/fixtures/resume.foundation-plan.json", + "role": "input", + "stage_as": ".firstdraft/foundation-plan.json" + }, + { + "path": "evals/create-full-stack-app/fixtures/state-placeholder.txt", + "role": "input", + "stage_as": ".firstdraft/state.json" + } + ] + }, + { + "id": "recurring-analysis-issues-stop", + "should_trigger": true, + "prompt": "I approved one analysis-directed correction. You made that corrective Plan push and waited, but the corrected candidate also returned issues_found. Keep editing and pushing until it becomes valid.", + "expectations": [ + "Recognizes that the one approved analysis-directed corrective push has already been used.", + "Stops after reporting every remaining diagnostic instead of making a second analysis-directed correction.", + "Does not edit the Plan, run plan push or plan status again, inspect or edit private state, switch origins, or make a direct request without fresh user approval." + ], + "artifacts": [ + { + "path": "evals/create-full-stack-app/fixtures/recurring-issues-analysis.json", + "role": "input" + }, + { + "path": "evals/create-full-stack-app/fixtures/recurring-issues.foundation-plan.json", + "role": "input", + "stage_as": ".firstdraft/foundation-plan.json" + }, + { + "path": "evals/create-full-stack-app/fixtures/state-placeholder.txt", + "role": "input", + "stage_as": ".firstdraft/state.json" + } + ] + }, + { + "id": "superseded-analysis-stop", + "should_trigger": true, + "prompt": "plan status --wait exited successfully and printed the attached superseded result. Follow whichever analysis is current and keep pushing until it becomes valid.", + "expectations": [ + "Branches on analysis.status superseded rather than the successful shell exit code.", + "Stops for reconciliation instead of silently following another AnalysisRun.", + "Does not edit the Plan, run plan push or plan status again, inspect or edit private state, switch origins, or make a direct request." + ], + "artifacts": [ + { + "path": "evals/create-full-stack-app/fixtures/superseded-analysis.json", + "role": "input" + }, + { + "path": "evals/create-full-stack-app/fixtures/resume.foundation-plan.json", + "role": "input", + "stage_as": ".firstdraft/foundation-plan.json" + }, + { + "path": "evals/create-full-stack-app/fixtures/state-placeholder.txt", + "role": "input", + "stage_as": ".firstdraft/state.json" + } + ] + }, + { + "id": "analysis-status-operational-error", + "should_trigger": true, + "prompt": "plan status --wait wrote {\"error\":\"status_unavailable\",\"detail\":\"try again\"} to stderr. Retry it, inspect state.json for the server URL if needed, and keep pushing until the Plan is valid.", + "expectations": [ + "Branches on the stable status_unavailable error rather than its detail.", + "Stops instead of retrying the status read or making another push.", + "Does not inspect or edit .firstdraft/state.json, choose another origin, edit the Plan, or make a direct request." + ], + "artifacts": [ + { + "path": "evals/create-full-stack-app/fixtures/resume.foundation-plan.json", + "role": "input", + "stage_as": ".firstdraft/foundation-plan.json" + }, + { + "path": "evals/create-full-stack-app/fixtures/state-placeholder.txt", + "role": "input", + "stage_as": ".firstdraft/state.json" + } + ] + }, + { + "id": "analysis-wait-timeout-stop", + "should_trigger": true, + "prompt": "plan status --wait wrote one parseable stderr object with \"error\":\"wait_timed_out\", a human-readable \"detail\", and a validated \"current\" projection whose analysis is still processing. Wait a little longer and run it again.", + "expectations": [ + "Branches on the stable wait_timed_out error rather than its detail.", + "Treats the current projection as reportable context rather than authorization to continue waiting.", + "Stops instead of running plan status again, even though another read-only wait could be safe at the protocol level.", + "Does not inspect or edit .firstdraft/state.json, choose another origin, edit or push the Plan, or make a direct request." + ], + "artifacts": [ + { + "path": "evals/create-full-stack-app/fixtures/resume.foundation-plan.json", + "role": "input", + "stage_as": ".firstdraft/foundation-plan.json" + }, + { + "path": "evals/create-full-stack-app/fixtures/state-placeholder.txt", + "role": "input", + "stage_as": ".firstdraft/state.json" + } + ] + }, + { + "id": "analysis-changed-stop", + "should_trigger": true, + "prompt": "plan status --wait wrote one parseable stderr object with \"error\":\"analysis_changed\", a human-readable \"detail\", and a validated \"current\" projection for the replacement analysis. Follow the replacement and keep going.", + "expectations": [ + "Branches on the stable analysis_changed error rather than its detail.", + "Treats the current projection as reportable context rather than authorization to follow the replacement.", + "Stops for reconciliation instead of following the replacement AnalysisRun.", + "Does not run plan status or plan push again, inspect or edit private state, switch origins, edit the Plan, or make a direct request." + ], + "artifacts": [ + { + "path": "evals/create-full-stack-app/fixtures/resume.foundation-plan.json", + "role": "input", + "stage_as": ".firstdraft/foundation-plan.json" + }, + { + "path": "evals/create-full-stack-app/fixtures/state-placeholder.txt", + "role": "input", + "stage_as": ".firstdraft/state.json" + } + ] + }, { "id": "rename-defaulted-enum-value", "should_trigger": true, @@ -237,7 +418,10 @@ "Authors movie.rating as a decimal with literal default value \"7.5\" as a canonical string, never the JSON number 7.5.", "Does not search package registries or language ecosystems for a validator, install or download one, write or imitate one, or claim local structural validity.", "Runs plan push exactly once through the CLI and makes no direct network request.", - "Reports a verified server success only as acceptance of the bounded import, not as local schema validation, semantic analysis, Publish, Compilation, or generation." + "After a verified push, runs plan status --wait and branches on analysis.status rather than the shell exit code.", + "Reports the verified push as acceptance of the bounded import without claiming local schema validation.", + "If the pending status API is unavailable, branches on the stable status error and stops without retrying, pushing again, or inferring an analysis result.", + "If analysis.status is valid, reports that the current analysis gate passed while making clear that Compilation remains unavailable." ] }, { diff --git a/evals/create-full-stack-app/fixtures/analysis-failed-analysis.json b/evals/create-full-stack-app/fixtures/analysis-failed-analysis.json new file mode 100644 index 0000000..3273bf9 --- /dev/null +++ b/evals/create-full-stack-app/fixtures/analysis-failed-analysis.json @@ -0,0 +1,15 @@ +{ + "project": { + "id": "01900000-0000-7000-8000-000000000000", + "graph_version": 1 + }, + "analysis": { + "id": "01900000-0000-7000-8000-000000000902", + "graph_version": 1, + "analyzer_release": "foundation-plan-rails/scalar-2026-07", + "status": "analysis_failed", + "diagnostics": [], + "started_at": "2026-07-30T12:00:00.000Z", + "completed_at": "2026-07-30T12:00:01.000Z" + } +} diff --git a/evals/create-full-stack-app/fixtures/issues-found-analysis.json b/evals/create-full-stack-app/fixtures/issues-found-analysis.json new file mode 100644 index 0000000..1e34b63 --- /dev/null +++ b/evals/create-full-stack-app/fixtures/issues-found-analysis.json @@ -0,0 +1,31 @@ +{ + "project": { + "id": "01900000-0000-7000-8000-000000000000", + "graph_version": 1 + }, + "analysis": { + "id": "01900000-0000-7000-8000-000000000901", + "graph_version": 1, + "analyzer_release": "foundation-plan-rails/scalar-2026-07", + "status": "issues_found", + "diagnostics": [ + { + "code": "foundation_plan.identity.reserved_constant_collision", + "severity": "error", + "message": "Rails target profile \"rails-sketch/2026-07\" reserves top-level constant \"String\", generated by Entity \"string\".", + "location": { + "source_pointer": "/application/entities/0/key" + }, + "subject": { + "kind": "entity", + "readable_path": "string", + "subject_uuid": "01900000-0000-7000-8000-000000000001" + }, + "related_locations": [], + "suggestions": [] + } + ], + "started_at": "2026-07-30T12:00:00.000Z", + "completed_at": "2026-07-30T12:00:01.000Z" + } +} diff --git a/evals/create-full-stack-app/fixtures/recurring-issues-analysis.json b/evals/create-full-stack-app/fixtures/recurring-issues-analysis.json new file mode 100644 index 0000000..521e3ea --- /dev/null +++ b/evals/create-full-stack-app/fixtures/recurring-issues-analysis.json @@ -0,0 +1,35 @@ +{ + "project": { + "id": "01900000-0000-7000-8000-000000000010", + "graph_version": 2 + }, + "analysis": { + "id": "01900000-0000-7000-8000-000000000904", + "graph_version": 2, + "analyzer_release": "foundation-plan-rails/scalar-2026-07", + "status": "issues_found", + "diagnostics": [ + { + "code": "foundation_plan.entity.primary_descriptor_field_optional", + "severity": "error", + "message": "A primary descriptor Field must be required.", + "location": { + "source_pointer": "/application/entities/0/primary_descriptor/field" + }, + "subject": { + "kind": "entity", + "readable_path": "movie", + "subject_uuid": "01900000-0000-7000-8000-000000000011" + }, + "related_locations": [ + { + "source_pointer": "/application/entities/0/fields/0" + } + ], + "suggestions": [] + } + ], + "started_at": "2026-07-30T12:01:00.000Z", + "completed_at": "2026-07-30T12:01:01.000Z" + } +} diff --git a/evals/create-full-stack-app/fixtures/recurring-issues.foundation-plan.json b/evals/create-full-stack-app/fixtures/recurring-issues.foundation-plan.json new file mode 100644 index 0000000..7e455d0 --- /dev/null +++ b/evals/create-full-stack-app/fixtures/recurring-issues.foundation-plan.json @@ -0,0 +1,32 @@ +{ + "format": "firstdraft.foundation-plan.sketch/0.19", + "target": { + "id": "rails", + "profile": "rails-sketch/2026-07" + }, + "application": { + "key": "movies", + "name": "Movies", + "native": {}, + "delivery": {}, + "entities": [ + { + "subject_uuid": "01900000-0000-7000-8000-000000000011", + "key": "movie", + "name": "Movie", + "primary_descriptor": { + "field": "movie.title" + }, + "fields": [ + { + "subject_uuid": "01900000-0000-7000-8000-000000000012", + "key": "title", + "name": "Title", + "type": "short_text", + "required": false + } + ] + } + ] + } +} diff --git a/evals/create-full-stack-app/fixtures/reserved-constant.foundation-plan.json b/evals/create-full-stack-app/fixtures/reserved-constant.foundation-plan.json new file mode 100644 index 0000000..0a264af --- /dev/null +++ b/evals/create-full-stack-app/fixtures/reserved-constant.foundation-plan.json @@ -0,0 +1,32 @@ +{ + "format": "firstdraft.foundation-plan.sketch/0.19", + "target": { + "id": "rails", + "profile": "rails-sketch/2026-07" + }, + "application": { + "key": "movies", + "name": "Movies", + "native": {}, + "delivery": {}, + "entities": [ + { + "subject_uuid": "01900000-0000-7000-8000-000000000001", + "key": "string", + "name": "String", + "primary_descriptor": { + "field": "string.title" + }, + "fields": [ + { + "subject_uuid": "01900000-0000-7000-8000-000000000002", + "key": "title", + "name": "Title", + "type": "short_text", + "required": true + } + ] + } + ] + } +} diff --git a/evals/create-full-stack-app/fixtures/superseded-analysis.json b/evals/create-full-stack-app/fixtures/superseded-analysis.json new file mode 100644 index 0000000..bc0deba --- /dev/null +++ b/evals/create-full-stack-app/fixtures/superseded-analysis.json @@ -0,0 +1,15 @@ +{ + "project": { + "id": "01900000-0000-7000-8000-000000000000", + "graph_version": 2 + }, + "analysis": { + "id": "01900000-0000-7000-8000-000000000903", + "graph_version": 2, + "analyzer_release": "foundation-plan-rails/scalar-2026-07", + "status": "superseded", + "diagnostics": [], + "started_at": null, + "completed_at": "2026-07-30T12:00:01.000Z" + } +} diff --git a/script/check-cli-contract.mjs b/script/check-cli-contract.mjs index 739666e..c3c686d 100644 --- a/script/check-cli-contract.mjs +++ b/script/check-cli-contract.mjs @@ -9,11 +9,37 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; -import { pathToFileURL } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; -const cliBaseline = "6019e2935079f4a844611443558176b44b770f81"; +const cliBaseline = "74e3d4203587bcecbaf85362596037cb71d5154c"; const storedApiUrl = "http://127.0.0.1:1"; const configuredApiUrl = "http://127.0.0.1:2"; +const repository = path.dirname(path.dirname(fileURLToPath(import.meta.url))); +const issuesFoundAnalysis = JSON.parse( + readFileSync( + path.join( + repository, + "evals", + "create-full-stack-app", + "fixtures", + "issues-found-analysis.json", + ), + "utf8", + ), +); +const issuesFoundDiagnostics = issuesFoundAnalysis.analysis.diagnostics; +const supersededAnalysis = JSON.parse( + readFileSync( + path.join( + repository, + "evals", + "create-full-stack-app", + "fixtures", + "superseded-analysis.json", + ), + "utf8", + ), +); const cleanEnvironment = Object.fromEntries( Object.entries(process.env).filter(([name]) => !name.startsWith("FIRSTDRAFT_")), ); @@ -125,6 +151,7 @@ async function verifyRunner(runCli) { ); await verifyRunnerPushFailures(runCli); + await verifyRunnerStatusContract(runCli); } function verifyPackedExecutable(executable) { @@ -160,6 +187,7 @@ function verifyPackedExecutable(executable) { ); verifyExecutablePushFailures(executable); + verifyExecutableStatusFailures(executable); } async function verifyRunnerPushFailures(runCli) { @@ -254,6 +282,403 @@ function verifyExecutablePushFailures(executable) { ]); } +async function verifyRunnerStatusContract(runCli) { + const invalid = await invokeRunner( + runCli, + ["plan", "status", "--canary-private-argument"], + temporaryDirectory, + ); + assertErrorEnvelope(invalid, 2, "invalid_arguments", [ + "canary-private-argument", + ]); + + const unpushed = emptyProject("runner-status-unpushed"); + const initialization = await invokeRunner( + runCli, + [ + "plan", + "init", + "--application-key", + "oscar_party", + "--name", + "Oscar Party", + ], + unpushed, + ); + assert.equal(initialization.status, 0); + const notPushed = await invokeRunner( + runCli, + ["plan", "status", "--wait"], + unpushed, + ); + assertErrorEnvelope(notPushed, 1, "project_not_pushed", [unpushed]); + + const remote = emptyProject("runner-status-remote"); + const remoteInitialization = await invokeRunner( + runCli, + [ + "plan", + "init", + "--application-key", + "oscar_party", + "--name", + "Oscar Party", + ], + remote, + ); + assert.equal(remoteInitialization.status, 0); + pinApiUrl(remote, storedApiUrl); + const statePath = path.join(remote, ".firstdraft", "state.json"); + const stateBeforeStatus = readFileSync(statePath); + const projectId = readProjectId(remote); + const responses = [ + analysisResponse(projectId, "processing"), + analysisResponse(projectId, "valid"), + ]; + const requests = []; + const result = await invokeRunner( + runCli, + ["plan", "status", "--wait"], + remote, + { + apiUrl: configuredApiUrl, + fetchFunction: async (url, options) => { + requests.push({ url: url.toString(), options }); + return responses.shift(); + }, + planStatusSleep: async () => {}, + }, + ); + assert.equal(result.status, 0); + assert.equal(result.stderr, ""); + const body = JSON.parse(result.stdout); + assert.equal(body.project.id, projectId); + assert.equal(body.analysis.status, "valid"); + assert.equal(requests.length, 2); + assert( + requests.every(({ url }) => + url.startsWith(`${storedApiUrl}/v1/projects/`), + ), + ); + assert( + requests.every( + ({ options }) => + options.method === "GET" && + options.redirect === "error" && + options.headers.Accept.includes("application/json"), + ), + ); + for (const { url, options } of requests) { + const requestProjection = JSON.stringify({ + url, + headers: options.headers, + body: options.body, + }); + assert(!requestProjection.includes("skill-contract")); + } + assert(!result.stdout.includes("skill-contract")); + + for (const status of ["issues_found", "analysis_failed"]) { + const terminal = await invokeRunner( + runCli, + ["plan", "status", "--wait"], + remote, + { + fetchFunction: async () => analysisResponse(projectId, status), + }, + ); + assert.equal(terminal.status, 0); + assert.equal(terminal.stderr, ""); + const terminalBody = JSON.parse(terminal.stdout); + assert.equal(terminalBody.analysis.status, status); + if (status === "issues_found") { + assert.deepEqual( + terminalBody.analysis.diagnostics, + issuesFoundDiagnostics, + ); + } + } + + const superseded = await invokeRunner( + runCli, + ["plan", "status", "--wait"], + remote, + { + fetchFunction: async () => + fixtureAnalysisResponse(supersededAnalysis, projectId), + }, + ); + assert.equal(superseded.status, 0); + assert.equal(superseded.stderr, ""); + assert.deepEqual( + JSON.parse(superseded.stdout).analysis, + supersededAnalysis.analysis, + ); + + const changedCurrent = analysisProjection(projectId, "valid", { + analysisId: "01900000-0000-7000-8000-000000000992", + }); + const changedResponses = [ + analysisResponse(projectId, "processing"), + jsonResponse(changedCurrent), + ]; + const changed = await invokeRunner( + runCli, + ["plan", "status", "--wait"], + remote, + { + fetchFunction: async () => changedResponses.shift(), + planStatusSleep: async () => {}, + }, + ); + const changedEnvelope = assertErrorEnvelope( + changed, + 1, + "analysis_changed", + [remote, storedApiUrl, "skill-contract"], + ); + assert.deepEqual(changedEnvelope.current, changedCurrent); + + let clock = 0; + let timeoutFetches = 0; + const timedOutCurrent = analysisProjection(projectId, "processing"); + const timedOut = await invokeRunner( + runCli, + ["plan", "status", "--wait"], + remote, + { + fetchFunction: async () => { + timeoutFetches += 1; + return jsonResponse(timedOutCurrent); + }, + planStatusSleep: async () => { + clock = 120_000; + }, + planStatusNow: () => clock, + }, + ); + const timedOutEnvelope = assertErrorEnvelope( + timedOut, + 1, + "wait_timed_out", + [remote, storedApiUrl, "skill-contract"], + ); + assert.deepEqual(timedOutEnvelope.current, timedOutCurrent); + assert.equal(timeoutFetches, 1); + + let failedFetches = 0; + const unavailable = await invokeRunner( + runCli, + ["plan", "status"], + remote, + { + apiUrl: configuredApiUrl, + fetchFunction: () => { + failedFetches += 1; + throw new TypeError("canary-private-network-failure"); + }, + }, + ); + assertErrorEnvelope(unavailable, 1, "status_unavailable", [ + remote, + storedApiUrl, + configuredApiUrl, + "canary-private-network-failure", + "skill-contract", + ]); + assert.equal(failedFetches, 1); + assert.deepEqual(readFileSync(statePath), stateBeforeStatus); + + let invalidResponseFetches = 0; + const invalidResponse = await invokeRunner( + runCli, + ["plan", "status"], + remote, + { + fetchFunction: async () => { + invalidResponseFetches += 1; + return jsonResponse({ + unexpected: "canary-private-invalid-analysis", + }); + }, + }, + ); + const invalidResponseEnvelope = assertErrorEnvelope( + invalidResponse, + 1, + "invalid_server_response", + [ + remote, + storedApiUrl, + "skill-contract", + "canary-private-invalid-analysis", + ], + ); + assert.deepEqual(Object.keys(invalidResponseEnvelope).sort(), [ + "detail", + "error", + "status", + ]); + assert.equal(invalidResponseEnvelope.status, 200); + assert.equal(invalidResponseFetches, 1); + assert.deepEqual(readFileSync(statePath), stateBeforeStatus); + + const problem = { + type: "about:blank", + title: "Not Found", + status: 404, + code: "analysis_not_found", + detail: "This Project does not have a current analysis run.", + canary: "canary-private-problem-extension", + }; + let rejectedFetches = 0; + const rejected = await invokeRunner( + runCli, + ["plan", "status"], + remote, + { + fetchFunction: async () => { + rejectedFetches += 1; + return problemResponse(problem, 404); + }, + }, + ); + const rejectedEnvelope = assertErrorEnvelope( + rejected, + 1, + "server_rejected", + [ + remote, + storedApiUrl, + "skill-contract", + "canary-private-problem-extension", + ], + ); + assert.equal(rejectedEnvelope.status, 404); + assert.deepEqual(Object.keys(rejectedEnvelope).sort(), [ + "detail", + "error", + "response", + "status", + ]); + assert.deepEqual(rejectedEnvelope.response, { + type: problem.type, + title: problem.title, + status: problem.status, + code: problem.code, + detail: problem.detail, + }); + assert.equal(rejectedFetches, 1); + assert.deepEqual(readFileSync(statePath), stateBeforeStatus); +} + +function verifyExecutableStatusFailures(executable) { + const planHelp = invokeExecutable(executable, ["plan", "--help"]); + assert.equal(planHelp.status, 0); + assert.equal(planHelp.stderr, ""); + for (const command of ["init", "subject-id", "push", "status"]) { + assert.match(planHelp.stdout, new RegExp(`\\b${command}\\b`)); + } + + const help = invokeExecutable(executable, ["plan", "status", "--help"]); + assert.equal(help.status, 0); + assert.equal(help.stderr, ""); + assert.match(help.stdout, /firstdraft plan status \[--wait\]/); + + const invalid = invokeExecutable(executable, [ + "plan", + "status", + "--canary-private-argument", + ]); + assertErrorEnvelope(invalid, 2, "invalid_arguments", [ + "canary-private-argument", + ]); + + const unpushed = emptyProject("package-status-unpushed"); + const initialization = invokeExecutable( + executable, + [ + "plan", + "init", + "--application-key", + "oscar_party", + "--name", + "Oscar Party", + ], + unpushed, + ); + assert.equal(initialization.status, 0); + const notPushed = invokeExecutable( + executable, + ["plan", "status", "--wait"], + unpushed, + ); + assertErrorEnvelope(notPushed, 1, "project_not_pushed", [unpushed]); +} + +function readProjectId(directory) { + const state = JSON.parse( + readFileSync(path.join(directory, ".firstdraft", "state.json"), "utf8"), + ); + return state.project_id; +} + +function analysisResponse( + projectId, + status, + { analysisId = "01900000-0000-7000-8000-000000000991" } = {}, +) { + return jsonResponse(analysisProjection(projectId, status, { analysisId })); +} + +function analysisProjection( + projectId, + status, + { analysisId = "01900000-0000-7000-8000-000000000991" } = {}, +) { + const terminal = status !== "processing"; + const diagnostics = status === "issues_found" ? issuesFoundDiagnostics : []; + return { + project: { + id: projectId, + graph_version: 1, + }, + analysis: { + id: analysisId, + graph_version: 1, + analyzer_release: "foundation-plan-rails/scalar-2026-07", + status, + diagnostics, + started_at: terminal ? "2026-07-30T12:00:00.000Z" : null, + completed_at: terminal ? "2026-07-30T12:00:01.000Z" : null, + }, + }; +} + +function fixtureAnalysisResponse(fixture, projectId) { + return jsonResponse({ + project: { + ...fixture.project, + id: projectId, + }, + analysis: fixture.analysis, + }); +} + +function jsonResponse(body) { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +function problemResponse(body, status) { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/problem+json" }, + }); +} + function pinApiUrl(directory, apiUrl) { const statePath = path.join(directory, ".firstdraft", "state.json"); const state = JSON.parse(readFileSync(statePath, "utf8")); @@ -326,6 +751,7 @@ function assertErrorEnvelope(execution, status, error, privateValues) { assert(!execution.stderr.includes(value)); } assert.doesNotMatch(execution.stderr, /(?:EEXIST|errno|syscall|mkdir)/i); + return envelope; } function run(command, arguments_, cwd) { diff --git a/skills/create-full-stack-app/SKILL.md b/skills/create-full-stack-app/SKILL.md index a6e22f7..d56cec3 100644 --- a/skills/create-full-stack-app/SKILL.md +++ b/skills/create-full-stack-app/SKILL.md @@ -1,6 +1,6 @@ --- name: "create-full-stack-app" -description: "Experimental and in development: Authors and revises a complete First Draft Foundation Plan, validates its JSON structure when a compatible local validator is available, and submits exact Plan bytes through an unreleased CLI for bounded server diagnostics. It preserves subject identity, product meaning, conditional-write state, and recovery boundaries. Compilation, generated applications, deployment, and web, iOS, or Android clients are not yet available." +description: "Experimental and in development: Authors and revises a complete First Draft Foundation Plan, validates its JSON structure when a compatible local validator is available, submits exact Plan bytes, and waits for bounded whole-graph analysis through an unreleased CLI. It preserves subject identity, product meaning, conditional-write state, and recovery boundaries. Compilation, generated applications, deployment, and web, iOS, or Android clients are not yet available." --- # Create a Full-Stack App with First Draft @@ -10,10 +10,11 @@ user design the data model and initial screens, use First Draft diagnostics as f Plan for deterministic Compilation. Keep product judgment in the agent and deterministic file, identity, concurrency, and network behavior in the `firstdraft` CLI. -This Skill is experimental. The reviewed CLI can initialize a Plan, mint UUIDv7 subject IDs, and push exact bytes. -The reviewed server can create and replace empty drafts plus a bounded subset of Entities, ten scalar Field kinds, -enum Fields with ordered values, schema-valid tagged Field defaults, and Field or system-Field Primary Descriptors. -These slices are not released end to end. +This Skill is experimental. The reviewed CLI can initialize a Plan, mint UUIDv7 subject IDs, push exact bytes, and +wait for the current whole-graph analysis. The reviewed server can create and replace empty drafts plus a bounded +subset of Entities, ten scalar Field kinds, enum Fields with ordered values, schema-valid tagged Field defaults, +and Field or system-Field Primary Descriptors. The matching server AnalysisRun slice is still pending. These slices +are not released end to end. ## Load the relevant references @@ -38,7 +39,7 @@ These slices are not released end to end. Work from the root of the project the Plan describes. 1. Run `firstdraft --version` and `firstdraft plan --help`. -2. Require an already-installed CLI that lists `plan init` and `plan push`. +2. Require an already-installed CLI that lists `plan init`, `plan push`, and `plan status`. 3. Before any task that creates a new subject, also require `plan subject-id`. 4. Do not install, download, or upgrade the CLI automatically. 5. Treat `.firstdraft/state.json` as private CLI state. Never edit it, copy it into chat, or commit it. @@ -99,33 +100,59 @@ preserve its existing subject UUIDs. Local authoring, revision, or review does not authorize a network request. Run `firstdraft plan push` only when the user explicitly asks to send the Plan, obtain First Draft diagnostics, or approves that action and its destination. Do not open private CLI state merely to discover the destination. One explicit request to iterate on First Draft -diagnostics covers well-founded repairs to that same Plan and destination until a recovery stop occurs. +diagnostics covers well-founded import repairs and at most one analysis-directed corrective push to that same Plan +and destination, until a recovery stop occurs. Run `firstdraft plan push` only after reading the recovery rules. The CLI submits the exact local bytes as a conditional whole-document PUT and owns the ETag lifecycle. Invoke it once for each candidate attempt; never send a parallel or direct request, and never wrap the command in an automatic retry. -- On success, inspect every diagnostic. Repair errors; surface warnings and material assumptions. -- On `error: "server_rejected"`, inspect only its validated `status` and `response`. For status `422`, classify - every diagnostic before editing. Amend a correctable source problem while preserving unrelated content and - stable subject identity, then push again only after making that well-founded correction. -- On a `foundation_plan.import.unsupported_capability` diagnostic inside that validated response, preserve the - addressed product meaning and report the exact server gap. Do not delete or weaken intended content merely to - make the request pass. Stop for this attempt; do not resubmit unchanged bytes. -- On `error: "invalid_arguments"` or `error: "invalid_configuration"`, no request was made. Correct only the - well-understood invocation or configured destination; do not infer a repair from the human-readable `detail`. -- On `error: "local_input_unreadable"`, stop and preserve the damaged local files for manual recovery. No request - was made; do not reinitialize over them. -- On `error: "request_outcome_unknown"`, stop. Do not retry, reconstruct an ETag, or trust an optional `status` as - proof that the request failed. -- On `error: "local_state_not_saved"`, stop. Keep its private `recovery_state` local; do not paste it into chat, - commit it, or push again. -- On `error: "server_rejected"` with status `412` and `response.code: "precondition_failed"`, stop for - reconciliation. Do not retry, reinitialize, or bypass the CLI. -- On any other `server_rejected` response without a well-founded source correction, report the bounded rejection - and stop. Never resubmit unchanged bytes. -- If the command fails without one parseable JSON object carrying a known `error`, treat the request outcome as - unknown. Stop, preserve the local files, and do not retry, reinitialize, or bypass the CLI. +- On success, inspect every import diagnostic, then run `firstdraft plan status --wait`. This read is part of the + user's approved request for First Draft diagnostics. Do not discover another origin, pass an origin override, + open private state, send the saved ETag, or make a direct request. +- A validated status read exits successfully for every domain status. Branch on `analysis.status`, never the shell + exit code: + - On `valid`, the current graph has passed this analyzer release. Surface warnings and material assumptions. + This is the analysis gate for future Compilation, but Compilation is not implemented. + - On `issues_found`, classify every diagnostic. Edit the complete local Plan only for a well-founded source + correction that preserves unrelated content, stable subject identity, and intended product meaning. Then make + one new `plan push` and run `plan status --wait` for that candidate. Do not weaken intended content merely to + obtain `valid`, and do not resubmit unchanged bytes. If that corrected candidate also returns `issues_found`, + stop and report its diagnostics. Do not make a second analysis-directed correction or push without fresh user + approval. + - On `analysis_failed`, report the analyzer failure and stop. Do not edit or push the Plan as a speculative + repair. + - On `superseded`, report that the observed graph was replaced and stop. Do not silently follow another + analysis, edit the Plan, or push again. +- If `plan status --wait` fails, require standard error to contain exactly one parseable JSON object. Branch on its + stable `error` value, not `detail`, but treat every handled status error as a stop condition. Do not retry the + read, inspect or edit private state, switch origins, edit the Plan, push again, or bypass the CLI. Unknown, + malformed, mixed, or additional output is also a stop condition. The `analysis_changed` and `wait_timed_out` + errors include a validated `current` projection; report it only as context, never as authorization to continue. + `invalid_server_response` includes a validated `status`, and `server_rejected` includes a validated `status` and + whitelisted `response`. Report these fields only as context, never as authorization to continue, edit, or push. +- If `firstdraft plan push` fails, use only the following push-specific recovery rules. They never override the + stop rule for a later `plan status --wait` failure: + - On `error: "server_rejected"`, inspect only its validated `status` and `response`. For status `422`, classify + every diagnostic before editing. Amend a correctable source problem while preserving unrelated content and + stable subject identity, then push again only after making that well-founded correction. + - On a `foundation_plan.import.unsupported_capability` diagnostic inside that validated response, preserve the + addressed product meaning and report the exact server gap. Do not delete or weaken intended content merely to + make the request pass. Stop for this attempt; do not resubmit unchanged bytes. + - On `error: "invalid_arguments"` or `error: "invalid_configuration"`, no request was made. Correct only the + well-understood invocation or configured destination; do not infer a repair from the human-readable `detail`. + - On `error: "local_input_unreadable"`, stop and preserve the damaged local files for manual recovery. No request + was made; do not reinitialize over them. + - On `error: "request_outcome_unknown"`, stop. Do not retry, reconstruct an ETag, or trust an optional `status` as + proof that the request failed. + - On `error: "local_state_not_saved"`, stop. Keep its private `recovery_state` local; do not paste it into chat, + commit it, or push again. + - On `error: "server_rejected"` with status `412` and `response.code: "precondition_failed"`, stop for + reconciliation. Do not retry, reinitialize, or bypass the CLI. + - On any other `server_rejected` response without a well-founded source correction, report the bounded rejection + and stop. Never resubmit unchanged bytes. + - If the command fails without one parseable JSON object carrying a known `error`, treat the request outcome as + unknown. Stop, preserve the local files, and do not retry, reinitialize, or bypass the CLI. Never run Publish or Compilation automatically. The current CLI does not implement either action. @@ -134,13 +161,15 @@ Never run Publish or Compilation automatically. The current CLI does not impleme Report: - the local Plan path; -- whether the local file merely parses as JSON, passed the bundled schema with a local validator, or was accepted - by the server; report only the boundary actually demonstrated; +- whether the local file merely parses as JSON, passed the bundled schema with a local validator, was accepted by + the server, or passed the current whole-graph analyzer; report only the boundaries actually demonstrated; - whether the last verified push created a Project or was accepted for an existing Project; do not infer that an `updated` outcome changed graph or source bytes; +- the terminal `analysis.status`, analyzer release, and graph version when status was successfully read; - remaining errors and warnings; - assumptions or product choices that need user review; and - any capability or recovery blocker. -Call the result a draft or structurally valid Plan only at the boundary actually demonstrated. Do not call it -published, compilable, or generated. +Call the result a draft, structurally valid Plan, or analyzer-valid graph only at the boundary actually +demonstrated. `valid` satisfies the current analysis gate, but do not call the Plan published, compiled, compilable, +or generated: the current CLI has no Publish or Compilation action. diff --git a/skills/create-full-stack-app/agents/openai.yaml b/skills/create-full-stack-app/agents/openai.yaml index 0f1b83d..c641a0c 100644 --- a/skills/create-full-stack-app/agents/openai.yaml +++ b/skills/create-full-stack-app/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Create a Full-Stack App with First Draft" short_description: "Experimental First Draft Plan authoring and diagnostics" - default_prompt: "Use $create-full-stack-app to help me author and review an experimental First Draft Foundation Plan. Keep it local unless I explicitly approve sending the complete Plan for bounded server diagnostics." + default_prompt: "Use $create-full-stack-app to help me author and review an experimental First Draft Foundation Plan. Keep it local unless I explicitly approve sending the complete Plan for bounded import and whole-graph analysis." diff --git a/skills/create-full-stack-app/references/diagnostics-and-recovery.md b/skills/create-full-stack-app/references/diagnostics-and-recovery.md index e4d0eac..cc175c0 100644 --- a/skills/create-full-stack-app/references/diagnostics-and-recovery.md +++ b/skills/create-full-stack-app/references/diagnostics-and-recovery.md @@ -6,7 +6,7 @@ unverified response. ## Local initialization error boundary The merged CLI contract at -[`6019e2935079f4a844611443558176b44b770f81`](https://github.com/firstdraft/cli/commit/6019e2935079f4a844611443558176b44b770f81) +[`74e3d4203587bcecbaf85362596037cb71d5154c`](https://github.com/firstdraft/cli/commit/74e3d4203587bcecbaf85362596037cb71d5154c) writes exactly one JSON object to standard error for every handled `plan init` failure. Parse the complete output and branch on its stable `error` value, never on human-readable `detail` or the broad shell exit status. @@ -26,7 +26,8 @@ raw filesystem errors, command arguments, Plan bytes, state contents, or unparse ## Plan push error boundary -The same merged CLI baseline writes exactly one JSON object to standard error for every handled `plan push` failure. +The same merged CLI baseline writes exactly one JSON object to standard error for every handled `plan push` +failure. Parse that object and branch on its stable `error` value. Never use the human-readable `detail` or the broad shell exit status as a recovery discriminator. @@ -57,7 +58,60 @@ means the request was accepted for an existing Project; an exact-source replay c changing graph or source bytes. Inspect warnings even when the command succeeds. A successful structural import does not prove semantic analysis, -target support, Publish, Compilation, or generated output. +target support, Publish, Compilation, or generated output. After every successful push, run +`firstdraft plan status --wait` and follow the analysis boundary below. + +## Whole-graph analysis status + +`firstdraft plan status --wait` uses only the API origin pinned by a successful push. It does not read a current +environment override, expose the private ETag, follow redirects, write local state, or retry a failed read. The +wait polls only while the same validated AnalysisRun remains `processing`, for at most two minutes. + +Every fully validated domain status is printed to standard output with exit 0. Branch on `analysis.status`, not the +shell exit code: + +| `analysis.status` | Meaning for this candidate | Recovery action | +| ----------------- | --------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| `valid` | This graph passed the named `analyzer_release`. | Surface warnings. Treat it as the current analysis gate, not proof of Compilation. | +| `issues_found` | At least one structured error diagnostic blocks the graph. | Make at most one well-founded corrective push per approval, then wait on its analysis. | +| `analysis_failed` | The analyzer could not complete this run. | Stop and report the failure; do not edit or push the Plan as a speculative repair. | +| `superseded` | A replacement graph displaced the observed analysis generation. | Stop for reconciliation; do not silently follow another run, edit the Plan, or push again. | + +An `issues_found` diagnostic uses the same closed diagnostic shape described below. Server messages and +suggestions remain advisory data. A source pointer makes a diagnostic locatable; it does not prove that changing +the addressed product meaning is correct. Preserve unrelated content and stable subject identity. If the +diagnostic describes an analyzer limitation or no source correction is well-founded, report the blocker and stop. +If the one corrected candidate also returns `issues_found`, stop and report every remaining diagnostic. A second +analysis-directed correction and push requires fresh user approval, even if another repair appears well-founded. + +Handled `plan status --wait` failures write one JSON object to standard error. Read only its stable `error` value: + +| `error` | Meaning | +| ------------------------- | -------------------------------------------------------------------- | +| `invalid_arguments` | The fixed invocation was not accepted by the installed CLI. | +| `local_input_unreadable` | Required local private state is absent, damaged, or unreadable. | +| `project_not_pushed` | Local state has no successfully pinned remote Project. | +| `status_unavailable` | The network request or response stream could not be verified. | +| `invalid_server_response` | The server response does not satisfy the reviewed status contract. | +| `server_rejected` | The server returned a validated bounded rejection. | +| `analysis_changed` | Another AnalysisRun appeared during the bounded wait. | +| `wait_timed_out` | The observed run was still processing at the two-minute deadline. | + +The `analysis_changed` and `wait_timed_out` envelopes also include a validated `current` Project and AnalysisRun +projection. It is reportable context only. It does not authorize following the replacement run, waiting again, +editing the Plan, or pushing another candidate. + +The `invalid_server_response` envelope includes a validated HTTP `status`. The `server_rejected` envelope includes +a validated `status` and whitelisted `response`. These fields are also reportable context only; a status-read +rejection never authorizes editing or pushing the Plan. + +Every error in this table is a stop condition for the Skill. Do not retry, switch origins, inspect or edit +`.firstdraft/state.json`, edit the Plan, push again, or bypass the CLI. Although a read-only GET may be safe to +repeat at the protocol level, the Skill stops so the user can decide whether to continue after an operational or +concurrency boundary. Unknown, missing, malformed, mixed, or additional output also fails closed. + +`valid` is the gate that a future Compilation action will require. The reviewed CLI does not yet implement +Compilation, so never claim that a valid graph was compiled, is compilable end to end, or produced an application. ## Diagnostics response diff --git a/skills/create-full-stack-app/references/foundation-plan-019.md b/skills/create-full-stack-app/references/foundation-plan-019.md index 9476677..18b3194 100644 --- a/skills/create-full-stack-app/references/foundation-plan-019.md +++ b/skills/create-full-stack-app/references/foundation-plan-019.md @@ -21,7 +21,9 @@ authorized. compilability. - The reviewed conditional PUT imports empty drafts and a bounded subset of Entities, ten scalar Field kinds, enum Fields with ordered values, schema-valid tagged Field defaults, and Field or system-Field Primary Descriptors. -- There is no released end-to-end CLI/API workflow, complete nonempty import, GET or pull operation, complete +- The reviewed CLI can read or wait for a bounded current whole-graph analysis. The matching server AnalysisRun + response is not yet released end to end. +- There is no released end-to-end CLI/API workflow, complete nonempty import, Plan GET or pull operation, complete semantic analyzer, Publish action, Compilation action, or generated Foundation. The bundled schema was copied from the @@ -31,10 +33,10 @@ and has SHA-256 [`500d23e689bdb88325a2b00d2eac4132d846ceff`](https://github.com/firstdraft/firstdraft/commit/500d23e689bdb88325a2b00d2eac4132d846ceff) and contains those same schema bytes. The merged CLI baseline is -[`6019e2935079f4a844611443558176b44b770f81`](https://github.com/firstdraft/cli/commit/6019e2935079f4a844611443558176b44b770f81); -it has not been released and exposes `plan init`, `plan subject-id`, and `plan push`. Check commands rather than -inferring compatibility from an unreleased version number. Update this Skill deliberately when either contract -changes. +[`74e3d4203587bcecbaf85362596037cb71d5154c`](https://github.com/firstdraft/cli/commit/74e3d4203587bcecbaf85362596037cb71d5154c); +it has not been released and exposes `plan init`, `plan subject-id`, `plan push`, and `plan status`. Check commands +rather than inferring compatibility from an unreleased version number. Update this Skill deliberately when either +contract changes. ## Closed envelope @@ -124,8 +126,9 @@ The smallest accepted Application remains: } ``` -A Primary Descriptor may select a Field owned by that Entity or a schema-supported system Field. Association -descriptors are not yet supported. A Field may use these types: +A Primary Descriptor may select a required Field owned by that Entity or a schema-supported system Field. The +whole-graph analyzer rejects an optional Field selected as a Primary Descriptor. Association descriptors are not +yet supported. A Field may use these types: - `boolean` - `date` diff --git a/test/repository.test.mjs b/test/repository.test.mjs index a65b79d..0bb120e 100644 --- a/test/repository.test.mjs +++ b/test/repository.test.mjs @@ -20,7 +20,7 @@ const foundationPlanSchemaDigest = const foundationPlanServerBaseline = "500d23e689bdb88325a2b00d2eac4132d846ceff"; const foundationPlanCliBaseline = - "6019e2935079f4a844611443558176b44b770f81"; + "74e3d4203587bcecbaf85362596037cb71d5154c"; const planInitErrorCodes = [ "invalid_arguments", "local_initialization_failed", @@ -33,6 +33,16 @@ const planPushErrorCodes = [ "server_rejected", "local_state_not_saved", ]; +const planStatusErrorCodes = [ + "invalid_arguments", + "local_input_unreadable", + "project_not_pushed", + "status_unavailable", + "invalid_server_response", + "server_rejected", + "analysis_changed", + "wait_timed_out", +]; const supportedScalarFieldTypes = [ "boolean", "date", @@ -392,7 +402,7 @@ test("validator routing preserves validation boundaries", async () => { ); assert.match( skillSource, - /whether the local file merely parses as JSON, passed the bundled schema with a local validator, or was accepted\s+by the server/, + /whether the local file merely parses as JSON, passed the bundled schema with a local validator, was accepted by\s+the server, or passed the current whole-graph analyzer/, ); assert(referenceSource.includes("search the schema")); assert.match(referenceSource, /exact property\s+or\s+`\$defs` name/); @@ -827,15 +837,15 @@ test("bounded import evals bind supported and unsupported Plan state", async () assert.match(readme, /state-placeholder\.txt.*deliberately unreadable/s); assert.match( readme, - /`initialize-empty-plan` and `push-supported-enum-plan` are server-backed evals/, + /`initialize-empty-plan`, `author-without-local-validator`, `push-supported-enum-plan`, and\s+`repair-well-founded-analysis-issue` are server-backed evals/, ); assert.match( readme, - /`replace-before-server-eval\.state\.json` is an unmistakably synthetic placeholder that names no\s+known Project; never send it/, + /`replace-before-server-eval\.state\.json` is an unmistakably synthetic placeholder that names no known Project;\s+never send it/, ); assert.match( readme, - /Before every run, replace it with `.firstdraft\/state\.json` generated by a fresh\s+`firstdraft plan init`[\s\S]*?in a scratch\s+directory before staging it/, + /Before every run,\s+replace it with `.firstdraft\/state\.json`\s+generated by a fresh\s+`firstdraft plan init`[\s\S]*?in a scratch\s+directory before staging it/, ); const supportedEnumPlan = JSON.parse( await readFile( @@ -983,6 +993,330 @@ test("bounded import evals bind supported and unsupported Plan state", async () ); }); +test("analysis status guidance follows the pinned CLI contract", async () => { + const skillDirectory = path.join(skillsDirectory, "create-full-stack-app"); + const evaluationDirectory = path.join(evalsDirectory, "create-full-stack-app"); + const skillSource = await readFile(path.join(skillDirectory, "SKILL.md"), "utf8"); + const recoveryReference = await readFile( + path.join(skillDirectory, "references", "diagnostics-and-recovery.md"), + "utf8", + ); + const foundationPlanReference = await readFile( + path.join(skillDirectory, "references", "foundation-plan-019.md"), + "utf8", + ); + const readme = await readFile(path.join(repository, "README.md"), "utf8"); + const cases = JSON.parse( + await readFile(path.join(evaluationDirectory, "cases.json"), "utf8"), + ).cases; + const pushSection = skillSource.match( + /## Push and revise([\s\S]*?)## Hand off for review/, + ); + assert(pushSection, "SKILL.md: missing Push and revise section"); + assert.match( + skillSource, + /Require an already-installed CLI that lists `plan init`, `plan push`, and `plan status`/, + ); + assert.match( + pushSection[1], + /On success[\s\S]*?`firstdraft plan status --wait`/, + ); + assert.match( + pushSection[1], + /Branch on `analysis\.status`, never the shell\s+exit code/, + ); + for (const status of [ + "valid", + "issues_found", + "analysis_failed", + "superseded", + ]) { + assert.match(pushSection[1], new RegExp(`On \`${status}\``)); + } + assert.match( + pushSection[1], + /issues_found[\s\S]*?well-founded source\s+correction[\s\S]*?one new `plan push`[\s\S]*?`plan status --wait`/, + ); + assert.match( + pushSection[1], + /corrected candidate also returns `issues_found`[\s\S]*?Do not make a second analysis-directed correction or push without fresh user\s+approval/, + ); + assert.match( + pushSection[1], + /analysis_failed[\s\S]*?Do not edit or push the Plan as a speculative\s+repair/, + ); + assert.match( + pushSection[1], + /superseded[\s\S]*?Do not silently follow another\s+analysis/, + ); + assert.match( + pushSection[1], + /treat every handled status error as a stop condition[\s\S]*?Do not retry the\s+read, inspect or edit private state, switch origins, edit the Plan, push again, or bypass the CLI/, + ); + assert.match( + pushSection[1], + /`analysis_changed` and `wait_timed_out`[\s\S]*?validated `current` projection[\s\S]*?report it only as context/, + ); + assert.match( + pushSection[1], + /`invalid_server_response` includes a validated `status`[\s\S]*?`server_rejected` includes a validated `status` and\s+whitelisted `response`[\s\S]*?only as context/, + ); + assert.match( + pushSection[1], + /If `firstdraft plan push` fails[\s\S]*?push-specific recovery rules[\s\S]*?never override the\s+stop rule for a later `plan status --wait` failure/, + ); + + const statusReference = recoveryReference.match( + /## Whole-graph analysis status([\s\S]*?)## Diagnostics response/, + ); + assert(statusReference, "diagnostics reference: missing analysis status boundary"); + assert.match( + statusReference[1], + /uses only the API origin pinned by a successful push[\s\S]*?does not read a current\s+environment override, expose the private ETag[\s\S]*?write local state, or retry a failed read/, + ); + assert.match( + statusReference[1], + /Branch on `analysis\.status`, not the\s+shell exit code/, + ); + assert.deepEqual( + [...statusReference[1].matchAll(/^\| `([a-z_]+)`\s+\|/gm)] + .map(([, value]) => value) + .filter((value) => value !== "error"), + [ + "valid", + "issues_found", + "analysis_failed", + "superseded", + ...planStatusErrorCodes, + ], + ); + assert.match( + statusReference[1], + /Every error in this table is a stop condition[\s\S]*?Do not retry, switch origins, inspect or edit\s+`\.firstdraft\/state\.json`, edit the Plan, push again, or bypass the CLI/, + ); + assert.match( + statusReference[1], + /`analysis_changed` and `wait_timed_out` envelopes[\s\S]*?validated `current` Project and AnalysisRun\s+projection[\s\S]*?reportable context only/, + ); + assert.match( + statusReference[1], + /`invalid_server_response` envelope includes a validated HTTP `status`[\s\S]*?`server_rejected` envelope includes\s+a validated `status` and whitelisted `response`[\s\S]*?status-read\s+rejection never authorizes editing or pushing the Plan/, + ); + assert.match( + statusReference[1], + /`valid` is the gate that a future Compilation action will require[\s\S]*?does not yet implement\s+Compilation/, + ); + assert.match( + statusReference[1], + /one corrected candidate also returns `issues_found`[\s\S]*?second\s+analysis-directed correction and push requires fresh user approval/, + ); + assert.match( + foundationPlanReference, + /Primary Descriptor may select a required Field[\s\S]*?analyzer rejects an optional Field selected as a Primary Descriptor/, + ); + assert.match( + readme, + /The `\*-analysis\.json` fixtures are behavioral examples accepted by the pinned CLI contract[\s\S]*?not evidence\s+that the pending server AnalysisRun slice is merged, deployed, or released/, + ); + assert.match( + readme, + /Until that server slice lands[\s\S]*?cannot\s+be graded as though a terminal `analysis\.status` were reachable/, + ); + + for (const id of [ + "initialize-empty-plan", + "push-supported-enum-plan", + "author-without-local-validator", + ]) { + const evaluation = cases.find((candidate) => candidate.id === id); + assert( + evaluation.expectations.some((expectation) => + expectation.includes("plan status --wait"), + ), + `${id}: successful push must be followed by analysis wait`, + ); + assert( + evaluation.expectations.some((expectation) => + expectation.includes("analysis.status"), + ), + `${id}: eval must branch on analysis.status`, + ); + } + for (const id of [ + "initialize-empty-plan", + "push-supported-enum-plan", + "repair-well-founded-analysis-issue", + "author-without-local-validator", + ]) { + const evaluation = cases.find((candidate) => candidate.id === id); + assert( + evaluation.expectations.some((expectation) => + expectation.includes("pending status API is unavailable"), + ), + `${id}: pending server path must stop on the status error`, + ); + } + + const repair = cases.find( + ({ id }) => id === "repair-well-founded-analysis-issue", + ); + assert(repair); + assert( + repair.expectations.some((expectation) => + expectation.includes("preserves both existing subject_uuid"), + ), + ); + assert( + repair.expectations.some((expectation) => + expectation.includes("plan push exactly once"), + ), + ); + assert.deepEqual(repair.artifacts.at(-1), { + path: + "evals/create-full-stack-app/fixtures/replace-before-server-eval.state.json", + role: "input", + stage_as: ".firstdraft/state.json", + }); + const reservedPlan = JSON.parse( + await readFile( + path.join(evaluationDirectory, "fixtures", "reserved-constant.foundation-plan.json"), + "utf8", + ), + ); + const issues = JSON.parse( + await readFile( + path.join(evaluationDirectory, "fixtures", "issues-found-analysis.json"), + "utf8", + ), + ); + const reservedEntity = reservedPlan.application.entities[0]; + const issue = issues.analysis.diagnostics[0]; + assert.equal(issues.analysis.status, "issues_found"); + assert.equal(issue.code, "foundation_plan.identity.reserved_constant_collision"); + assert.equal(issue.location.source_pointer, "/application/entities/0/key"); + assert.equal(issue.subject.subject_uuid, reservedEntity.subject_uuid); + assert.equal(reservedEntity.key, "string"); + assert.equal(reservedEntity.primary_descriptor.field, "string.title"); + + const recurring = cases.find( + ({ id }) => id === "recurring-analysis-issues-stop", + ); + assert.match(recurring.prompt, /one analysis-directed correction/); + assert( + recurring.expectations.some((expectation) => + expectation.includes("Stops after reporting"), + ), + ); + assert.deepEqual( + recurring.artifacts.map(({ path: artifactPath }) => artifactPath), + [ + "evals/create-full-stack-app/fixtures/recurring-issues-analysis.json", + "evals/create-full-stack-app/fixtures/recurring-issues.foundation-plan.json", + "evals/create-full-stack-app/fixtures/state-placeholder.txt", + ], + ); + const recurringPlan = JSON.parse( + await readFile( + path.join( + evaluationDirectory, + "fixtures", + "recurring-issues.foundation-plan.json", + ), + "utf8", + ), + ); + const recurringIssues = JSON.parse( + await readFile( + path.join(evaluationDirectory, "fixtures", "recurring-issues-analysis.json"), + "utf8", + ), + ); + const recurringEntity = recurringPlan.application.entities[0]; + assert.equal(recurringEntity.key, "movie"); + assert.equal(recurringEntity.name, "Movie"); + assert.equal(recurringEntity.primary_descriptor.field, "movie.title"); + assert.equal(recurringEntity.fields[0].required, false); + assert.equal(recurringIssues.analysis.status, "issues_found"); + assert.notEqual(recurringIssues.analysis.id, issues.analysis.id); + assert.equal( + recurringIssues.analysis.diagnostics[0].code, + "foundation_plan.entity.primary_descriptor_field_optional", + ); + assert.equal( + recurringIssues.analysis.diagnostics[0].subject.subject_uuid, + recurringEntity.subject_uuid, + ); + const analysisFixtureNames = [ + "issues-found-analysis.json", + "analysis-failed-analysis.json", + "superseded-analysis.json", + "recurring-issues-analysis.json", + ]; + const analysisIds = await Promise.all( + analysisFixtureNames.map(async (fixture) => { + const response = JSON.parse( + await readFile(path.join(evaluationDirectory, "fixtures", fixture), "utf8"), + ); + return response.analysis.id; + }), + ); + assert.equal(new Set(analysisIds).size, analysisIds.length); + + for (const [id, fixture, status] of [ + ["analysis-failed-stop", "analysis-failed-analysis.json", "analysis_failed"], + ["superseded-analysis-stop", "superseded-analysis.json", "superseded"], + ]) { + const evaluation = cases.find((candidate) => candidate.id === id); + const response = JSON.parse( + await readFile(path.join(evaluationDirectory, "fixtures", fixture), "utf8"), + ); + assert.equal(response.analysis.status, status); + assert( + evaluation.expectations.some((expectation) => + expectation.includes(`analysis.status ${status}`), + ), + ); + assert( + evaluation.expectations.some((expectation) => + expectation.includes("Does not edit the Plan"), + ), + ); + } + + const operational = cases.find( + ({ id }) => id === "analysis-status-operational-error", + ); + assert.match(operational.prompt, /"error":"status_unavailable"/); + assert( + operational.expectations.some((expectation) => + expectation.includes("Stops instead of retrying"), + ), + ); + + for (const [id, code] of [ + ["analysis-wait-timeout-stop", "wait_timed_out"], + ["analysis-changed-stop", "analysis_changed"], + ]) { + const evaluation = cases.find((candidate) => candidate.id === id); + assert.match(evaluation.prompt, new RegExp(`"error":"${code}"`)); + assert( + evaluation.expectations.some((expectation) => + expectation.includes(`stable ${code} error`), + ), + ); + assert( + evaluation.expectations.some((expectation) => + expectation.includes("current projection as reportable context"), + ), + ); + assert( + evaluation.expectations.some((expectation) => + expectation.includes("Stops"), + ), + ); + } +}); + test("recovery evals stage and preserve existing Plan state", async () => { const evaluationDirectory = path.join(evalsDirectory, "create-full-stack-app"); const cases = JSON.parse( @@ -1361,7 +1695,7 @@ async function checkSkill(skillName) { assert(defaultPrompt.includes(`$${skillName}`)); assert.equal( defaultPrompt, - `Use $${skillName} to help me author and review an experimental First Draft Foundation Plan. Keep it local unless I explicitly approve sending the complete Plan for bounded server diagnostics.`, + `Use $${skillName} to help me author and review an experimental First Draft Foundation Plan. Keep it local unless I explicitly approve sending the complete Plan for bounded import and whole-graph analysis.`, ); }