{ "schema_version": 2, "title": "Supervising coding agents without lying to yourself", "summary": "Nine guardrails for work that looks done and is not.", "canonical_source": "https://fieldnotes.clayj.app/field-notes.md", "authorship": { "name": "Clayton Johnson", "statement": "Written by Clayton Johnson while building Toolroll, a snippet vault for iPhone and iPad. Every failure recorded here happened there. clay@clayj.app", "toolroll_url": "https://toolroll.clayj.app", "contact_email": "clay@clayj.app" }, "generated_from": [ "README.md", "scripts/build.py" ], "flow": [ { "id": "dispatch", "number": "01", "title": "Separate the hands before work begins.", "summary": "Each worker receives its own copy of the repository. Its state lives on disk, so a crash, restart, or closed laptop does not erase what happened.", "detail": "Firstmate reconciles the files and repository state that actually exist instead of trusting an in-memory story. Isolation prevents two workers from editing the same files." }, { "id": "work", "number": "02", "title": "Questions go up. They do not fold back in.", "summary": "A worker can act inside its authority. A decision above that boundary becomes a durable stopping state routed to the supervisor.", "detail": "The worker cannot approve its own escalation, and an interface must not guess when the necessary context does not exist." }, { "id": "review", "number": "03", "title": "Change who is looking, then change what they look at.", "summary": "Review crosses model families on purpose. It also leaves the diff and walks the product as a stranger would.", "detail": "A third related finding stops the patch cycle and sends the problem back to the design model." }, { "id": "validate", "number": "04", "title": "Make evidence prove it can fail.", "summary": "The validation pipeline may refuse the merge. A new test is trusted only after it fails when the behaviour it protects is removed.", "detail": "Green is useful only after the red state demonstrates that the test observes the production seam." }, { "id": "land", "number": "05", "title": "Believe the repository, not the narration.", "summary": "A passing pipeline is input to a decision, not the decision itself. Landing is reported only after the merged state is verified.", "detail": "Release identity moves forward too: reusing a build number can make a new binary behave like an old one." }, { "id": "teardown", "number": "06", "title": "Cleanup must prove what it will remove.", "summary": "Teardown refuses to proceed when saved work or current ownership cannot be established.", "detail": "A safety refusal is a stop-and-investigate result. It is not an error to retry past." } ], "claims": [ { "id": "dispatch", "number": "01", "label": "Isolation and durable state", "title": "Every worker gets a separate world.", "summary": "Tasks begin in separate copies of the repository. Worker state is durable on disk, so a crash, restart, or closed laptop does not erase what happened.", "bullets": [ "Separate files. Separate branches.", "Restart from disk, not from memory.", "Reconcile what exists before acting." ], "guardrails": [] }, { "id": "work", "number": "02", "label": "Authority boundary", "title": "Questions go up. Work waits.", "summary": "A worker can act inside its authority. A question above that boundary becomes a durable stopping state routed to the supervisor.", "bullets": [ "The worker cannot answer itself.", "The exact open decision must be resolved.", "When context is unknowable, say so." ], "guardrails": [ 3, 8 ] }, { "id": "review", "number": "03", "label": "Independent review", "title": "Models check each other.", "summary": "Firstmate can run Claude, Codex, Cursor, Grok, OpenCode, Pi, Kimi, and Muse. The useful part is not the list. Review goes to a different model family, and work routes across providers with remaining quota.", "bullets": [ "Different family from the implementer.", "Remaining quota routes the next job.", "Product review leaves the diff behind." ], "guardrails": [ 4, 5, 6 ] }, { "id": "validate", "number": "04", "label": "Validation and landing", "title": "Green is a request for proof.", "summary": "The validation pipeline can refuse a merge. A test has to demonstrate that it can fail, and a merge is never reported as landed without evidence from the repository.", "bullets": [ "Red proves the test can see the seam.", "Green checks are not a merge argument.", "Release identity only moves forward." ], "guardrails": [ 1, 2, 9 ] }, { "id": "teardown", "number": "05", "label": "Safe teardown", "title": "Teardown gets one chance to be right.", "summary": "Cleanup refuses to proceed when saved work or current ownership cannot be proved. A safety refusal is a result, not an invitation to keep retrying.", "bullets": [ "Unsaved work stops cleanup.", "Current ownership is checked again.", "Nothing is discarded on inference." ], "guardrails": [ 7 ] } ], "attention_sample": { "id": "attention", "section_title": "A day at the helm", "date": "2026-08-31", "date_label": "31 August 2026", "scope": "Toolroll fleet", "closed_records": 11, "closed_record_breakdown": { "merged_pull_requests": 6, "review_and_design_records": 3, "local_corrections": 2 }, "captain_replies": 2, "counting_rule": "Distinct captain replies are counted once, including when one reply routes to more than one task.", "decision_context": "The keyboard could not reach a snippet a user asked to paste. A worker proposed one message for everyone.", "decision_question": "Should a customer who has already paid be told to pin the snippet, when pinning does nothing for them and implies their purchase is not working?", "decision_answer": "No. A paying customer gets a plain statement that the snippet cannot be reached right now. The pin remedy is offered only to someone it would actually help.", "decision_boundary": "That distinction decided what a person was told about something they bought. No agent should make it.", "source": "Firstmate durable backlog and done archive: dated closures, merged pull-request records, and captain-hold resolutions." }, "guardrails": [ { "id": "guardrail-1", "number": 1, "title": "A test must fail before you trust it passing", "stage": "validate", "where": "Validation gate", "rule": "Before accepting any new test, confirm it fails against the broken behaviour it claims to cover. If it passes both ways, it is decoration.", "caught": "This project shipped tests that pass whether or not their subject exists — three separate times before anyone noticed the pattern. The clearest case: a test meant to prove that cancelling a compose screen saves nothing. It swapped one draft for another and compared storage. It never pressed Cancel. A reviewer put it exactly right: \"A regression where those actions call create or editBody would still pass.\" The test was testing that a test did nothing." }, { "id": "guardrail-2", "number": 2, "title": "Green checks are not an argument for merging", "stage": "land", "where": "Merge decision and landing proof", "rule": "CI proves the code compiles and existing tests pass. It cannot prove the change does what its description says. Read the diff.", "caught": "A status reader reported a task as done · PR merged/closed. No pull request existed — the pipeline had skipped creating one because a credential had expired, and only a branch had been pushed. Trusting that reading would have meant reporting shipped work that was still sitting on a branch." }, { "id": "guardrail-3", "number": 3, "title": "The worker never answers its own escalation", "stage": "work", "where": "Worker authority boundary", "rule": "When an agent hits a question above its authority, it stops and asks. It does not decide, and it does not decide and then mention it.", "caught": "A review gate asked whether a test seam was acceptable. The worker escalated rather than approving its own work — and was right to, because the answer changed the design. An agent that resolves its own blockers produces work nobody reviewed at the moment review mattered." }, { "id": "guardrail-4", "number": 4, "title": "Two rounds is diligence, three is a design problem", "stage": "review", "where": "Repeated review findings", "rule": "Set a stop condition before you need it. If review raises a third finding in the same theme, stop patching and look at the model.", "caught": "Three separate defects, each fixed individually: Setting a pin to the value it already held cleared a snippet's ownership. Appending an empty string cleared it too. Saving a form without changing anything retired a first-run state. Three fixes, one cause: each consumer decided for itself whether something had changed, instead of being told by the mutation that performed it. Patch one caller and the next repeats the mistake. The stop condition caught it on the third round. Without it there would have been a fourth, a fifth, and a codebase full of guards." }, { "id": "guardrail-5", "number": 5, "title": "Review from a different model family", "stage": "review", "where": "Review assignment", "rule": "Have work reviewed by a model that did not write it and does not share its training. Independence matters more than capability.", "caught": "Three same-family code reviews passed over an app that had no way to create a record inside it. Every one checked features against their own briefs. None opened the product cold. A cross-family review found it in one pass — along with a privacy policy describing a keyboard the app did not ship." }, { "id": "guardrail-6", "number": 6, "title": "Walk the product like a stranger", "stage": "review", "where": "Product-level review", "rule": "Periodically use the thing as someone who has never seen it. Not as a test: open it, and try to do the obvious first thing.", "caught": "See above. The missing create button was found by the person who owns the product simply opening it — after three reviews and weeks of work by agents that had read every file. Reviewing a diff answers \"is this change correct.\" It cannot answer \"is this product sensible.\"" }, { "id": "guardrail-7", "number": 7, "title": "Never discard work you cannot prove is saved", "stage": "teardown", "where": "Worker teardown", "rule": "Refuse teardown, force, reset, and stash by default. A refusal citing a safety reason is a stop-and-investigate result, not an obstacle to retry past.", "caught": "A worktree was returned to a pool and immediately reassigned. A retried teardown then killed a different, live worker occupying the same slot. Nothing was lost that time, by luck." }, { "id": "guardrail-8", "number": 8, "title": "Prefer explanation to inference when the model can't know", "stage": "work", "where": "Feature and interface design", "rule": "When a feature cannot behave correctly in a given context, say so plainly rather than guessing at the user's intent.", "caught": "Snippets could contain fill-in placeholders that the keyboard replaces on insertion. Copying one in the app pasted raw {ask:Time} at whoever received it. The tempting fix was to teach Copy to resolve placeholders. But a fill-in needs a cursor to position, and Copy has none — there was no correct value to substitute. The right fix was to relabel the action Copy Exact Text and explain what it does. The button now makes an accurate promise to someone who reads nothing else." }, { "id": "guardrail-9", "number": 9, "title": "Version identity is functional, not cosmetic", "stage": "land", "where": "Release preparation", "rule": "Never reuse a build identifier.", "caught": "A build installed with the same version and build number as its predecessor caused iOS to serve stale App Intents metadata. Siri did not learn the app's actions at all until the phone was restarted. An hour went into diagnosing what looked like a broken feature and was a recycled number." } ] }