{ "date": "2026-09-08", "basis": "Local working tree including uncommitted edits; not a production or execution report", "base_commit": "fd64da70de5c92a08d625a60c4156b771e2bca13", "suites": [ { "path": "src/workflows/build-alt/steps/filter-content/prompts/filter-content", "status": "active", "rows": 4, "sample_sha256": "c1cd1872837a679e5e0b7c52f600e804b63e7aacbd1cd59ba54ec12ccf537b61" }, { "path": "src/workflows/build-alt/steps/generate-profile/prompts/communication-style", "status": "active", "rows": 4, "sample_sha256": "a2aaa5c337b23c4c7647d1c763ecad99c879d25829c4a087a3f9fcee7cc39392" }, { "path": "src/workflows/build-alt/steps/generate-profile/prompts/personality-profile", "status": "active", "rows": 6, "sample_sha256": "f85869b9c496cdc7d9645105b8d1aaa53c321b54344078e9dd04a534deb206ca" }, { "path": "src/workflows/build-alt/steps/memory-analyzer/prompts/employee-memory-extractor", "status": "active", "rows": 2, "sample_sha256": "026adb2ffd13fb601b9249bad2dbe49b24a59359a8e66a8c53da6c9b6cef2ea4" }, { "path": "src/workflows/build-alt/steps/run-web-research/prompts/run-web-research", "status": "excluded", "rows": 8, "sample_sha256": "7082a81c69d6451a97b7700b4c2b8ddb8defeb626543eab4878c97c6623a0a26" }, { "path": "src/workflows/complete-submission/steps/capture-and-store/prompts/capture-submission", "status": "active", "rows": 6, "sample_sha256": "53392cfae838f500985c2210e387fea1a510b413051abba9e8fdc2546cf2558e" }, { "path": "src/workflows/create-job/prompts/external-master-prompt", "status": "excluded", "rows": 22, "sample_sha256": "4d8d0b5d15d6ab020f18244b23e2fa762947e5ff0279d4f87cc962b91d4cbcf3" }, { "path": "src/workflows/create-job/steps/await-setup-completion/prompts/job-and-interview-setup", "status": "excluded", "rows": 6, "sample_sha256": "afcc43e46d82b8b784c5bbe72c6cd901cb29e903593aca83b9559c04b3d4c1e9" }, { "path": "src/workflows/derive-query-surface/prompts/write-query-briefing", "status": "active", "rows": 4, "sample_sha256": "0b503d676982f636abe2e12b6dea0373ec88bce7fa7a5697d12d65fadfe560be" }, { "path": "src/workflows/form/handoff-briefs/prompts/chatgpt", "status": "active", "rows": 9, "sample_sha256": "e39723a8f743eb19b9218c6bc9ba3e63b3b3d15346801030a88fe630e6587030" }, { "path": "src/workflows/form/handoff-briefs/prompts/claude", "status": "active", "rows": 7, "sample_sha256": "fdacd70573e56f6c4acdb04d0e2849f8d5be879244c7f7e50b15f18b888648e1" }, { "path": "src/workflows/form/prompts/form-agent", "status": "active", "rows": 153, "sample_sha256": "57dc28cc3bdeb807effec88cee78c65700d97a01c4d1a39b7f06ffb163c97f37" }, { "path": "src/workflows/form/prompts/form-master-prompt", "status": "excluded", "rows": 4, "sample_sha256": "c0694bc35eea951ce2852057140bccb31c2682caf9e2a209a66b7b0e06366e10" }, { "path": "src/workflows/form/prompts/query-agent", "status": "active", "rows": 23, "sample_sha256": "43ae13215d90dd3ca6dbb5cc81dc7bd7c82bb0c90d1afd7fe3db439dbc5a725d" }, { "path": "src/workflows/index-submission/prompts/submission-claim-extractor", "status": "active", "rows": 9, "sample_sha256": "51c5f2ac3c5f01eeb936dfc25f3fd98ac72ca4bb2556e4e7f12454c4371f5265" }, { "path": "src/workflows/onboard-org/steps/get-jobs/prompts/get-jobs", "status": "active", "rows": 2, "sample_sha256": "66c5d67faf704dce4ae95d48fd42ac4b8a2c88f5fab744eb5b69718e0505ae9e" }, { "path": "src/workflows/onboard-org/steps/run-web-research/prompts/run-web-research", "status": "active", "rows": 4, "sample_sha256": "8b5183865ea3b5499b8e1da51d7501b8d8d5c43fa1bc5ad6e108d3d2d781a8dd" }, { "path": "src/workflows/onboard-org/steps/synthesize-org-memory/prompts/synthesize-org-memory", "status": "active", "rows": 4, "sample_sha256": "64df15e263a72044ec3b40729ea533dfb068256396dcfbf4cf1c99f0f92fb283" }, { "path": "src/workflows/prescreen/steps/evaluate-transcript/prompts/evaluate-prescreen-transcript", "status": "active", "rows": 3, "sample_sha256": "55cc26ad891d7753f9f0a5f7d310ba3b1196c66d763a8d724948674a8a5ef461" } ], "excluded_prompt_directories": [ "src/workflows/build-alt/prompts/extract-head", "src/workflows/build-alt/steps/animation-sprite-sheet/prompts/motion-blink", "src/workflows/build-alt/steps/animation-sprite-sheet/prompts/motion-breathing", "src/workflows/build-alt/steps/animation-sprite-sheet/prompts/motion-first-build-wave", "src/workflows/build-alt/steps/animation-sprite-sheet/prompts/motion-nod", "src/workflows/build-alt/steps/animation-sprite-sheet/prompts/motion-wave", "src/workflows/build-alt/steps/build-alt-prompt/prompts/build-alt-prompt", "src/workflows/build-alt/steps/build-alt-prompt/prompts/employee-alt", "src/workflows/build-alt/steps/describe-photo/prompts/describe-photo", "src/workflows/build-alt/steps/generate-pixel-art/prompts/generate-pixel-art", "src/workflows/build-alt/steps/generate-profile/prompts/refresh-alt-personality", "src/workflows/build-alt/steps/refine-pixel-art/prompts/refine-pixel-art", "src/workflows/build-alt/steps/run-web-research/prompts/run-web-research", "src/workflows/create-job/prompts/external-master-prompt", "src/workflows/create-job/steps/await-create-jd-completion/prompts/requirement-gathering", "src/workflows/create-job/steps/await-setup-completion/prompts/job-and-interview-setup", "src/workflows/form/prompts/form-master-prompt", "src/workflows/freeform-chat/prompts/alt-freeform", "src/workflows/generate-form-plate/steps/generate-plate/prompts/generate-plate", "src/workflows/onboard-org/steps/generate-org-background/prompts/generate-org-background", "src/workflows/prescreen/steps/compose-candidate-brief/prompts/compose-candidate-brief", "src/workflows/shared/prompts/interviewer", "src/workflows/sync-invited-user/steps/enrich-employee-profile/prompts/enrich-employee-profile", "src/workflows/update-org-background/prompts/generate-org-background-userprompt" ], "workflow_definitions": [ "src/workflows/add-user-to-org/definition.ts", "src/workflows/assign-user-to-job/definition.ts", "src/workflows/build-alt/definition.ts", "src/workflows/build-corpus-digest/definition.ts", "src/workflows/close-job-record/definition.ts", "src/workflows/complete-submission/definition.ts", "src/workflows/connect-account/definition.ts", "src/workflows/connect-slack/definition.ts", "src/workflows/create-job/definition.ts", "src/workflows/delete-org/definition.ts", "src/workflows/derive-query-surface/definition.ts", "src/workflows/enrich-employee-profile/definition.ts", "src/workflows/extract-alt-memory-from-file/definition.ts", "src/workflows/extract-org-memory-from-file/definition.ts", "src/workflows/freeform-chat/definition.ts", "src/workflows/generate-form-plate/definition.ts", "src/workflows/import-external-transcript/definition.ts", "src/workflows/index-submission/definition.ts", "src/workflows/invite-user/definition.ts", "src/workflows/onboard-org/definition.ts", "src/workflows/prescreen/definition.ts", "src/workflows/remove-user-from-job/definition.ts", "src/workflows/remove-user/definition.ts", "src/workflows/sync-invited-user/definition.ts", "src/workflows/sync-removed-user/definition.ts", "src/workflows/sync-user-role-change/definition.ts", "src/workflows/update-final-prompt-component/definition.ts", "src/workflows/update-org-background/definition.ts", "src/workflows/update-user-role/definition.ts" ], "operation_contracts": [ "src/operations/acknowledge-pre-interview-briefing/contract.ts", "src/operations/append-candidate-to-sheet/contract.ts", "src/operations/check-org-name-available/contract.ts", "src/operations/close-form-definition/contract.ts", "src/operations/complete-connector-setup/contract.ts", "src/operations/create-form-definition/contract.ts", "src/operations/delete-alt-animation/contract.ts", "src/operations/delete-alt-personality/contract.ts", "src/operations/enable-query-surface/contract.ts", "src/operations/export-form-submissions/contract.ts", "src/operations/finalize-external-job-interview-setup/contract.ts", "src/operations/get-admin-alt-personality/contract.ts", "src/operations/get-admin-alt/contract.ts", "src/operations/get-admin-employee-snapshots/contract.ts", "src/operations/get-admin-employee/contract.ts", "src/operations/get-admin-final-prompt-breakdown/contract.ts", "src/operations/get-admin-interview-config-by-job/contract.ts", "src/operations/get-admin-interview-config/contract.ts", "src/operations/get-admin-stats/contract.ts", "src/operations/get-alt-memory-extraction-prompt/contract.ts", "src/operations/get-alt-personality-authoring-prompt/contract.ts", "src/operations/get-alt-personality-refresh-prompt/contract.ts", "src/operations/get-chat-session-by-workflow/contract.ts", "src/operations/get-chat-session-status/contract.ts", "src/operations/get-employer-alt-sprite/contract.ts", "src/operations/get-employer-identity/contract.ts", "src/operations/get-employer-interview-artifact-url/contract.ts", "src/operations/get-form-definition/contract.ts", "src/operations/get-form-master-prompt/contract.ts", "src/operations/get-form-submission-transcript/contract.ts", "src/operations/get-form-submission/contract.ts", "src/operations/get-interview-evaluation/contract.ts", "src/operations/get-interview-transcript/contract.ts", "src/operations/get-master-prompt/contract.ts", "src/operations/get-onboarding-eligibility/contract.ts", "src/operations/get-org-memory-synthesis-prompt/contract.ts", "src/operations/get-public-employee-profile/contract.ts", "src/operations/get-public-form-metadata/contract.ts", "src/operations/get-public-job/contract.ts", "src/operations/get-public-org-alt/contract.ts", "src/operations/get-public-org-applicables/contract.ts", "src/operations/get-public-org-job/contract.ts", "src/operations/get-public-org-with-jobs/contract.ts", "src/operations/get-public-orgs/contract.ts", "src/operations/get-recent-call-context/contract.ts", "src/operations/get-workflow-status/contract.ts", "src/operations/list-active-jobs/contract.ts", "src/operations/list-admin-alt-memory-buckets/contract.ts", "src/operations/list-admin-alt-sprites/contract.ts", "src/operations/list-admin-alts/contract.ts", "src/operations/list-admin-employee-personality-files/contract.ts", "src/operations/list-admin-employee-sprite-source-images/contract.ts", "src/operations/list-admin-employees/contract.ts", "src/operations/list-admin-interview-configs-by-org/contract.ts", "src/operations/list-admin-interviews/contract.ts", "src/operations/list-admin-jobs/contract.ts", "src/operations/list-admin-org-memory-buckets/contract.ts", "src/operations/list-admin-orgs/contract.ts", "src/operations/list-admin-prompt-versions/contract.ts", "src/operations/list-candidates-for-job/contract.ts", "src/operations/list-chat-turns/contract.ts", "src/operations/list-form-definitions/contract.ts", "src/operations/list-form-submissions/contract.ts", "src/operations/manage-alt-memory-bucket/contract.ts", "src/operations/manage-org-memory-bucket/contract.ts", "src/operations/mint-admin-asset-read-credentials/contract.ts", "src/operations/mint-admin-org-asset-read-credentials/contract.ts", "src/operations/open-form-definition/contract.ts", "src/operations/prepare-build-alt-upload/contract.ts", "src/operations/prepare-external-job-interview-setup/contract.ts", "src/operations/prepare-memory-upload/contract.ts", "src/operations/prepare-sprite-source-upload/contract.ts", "src/operations/presign-admin-asset-read/contract.ts", "src/operations/register-personality-source/contract.ts", "src/operations/register-sprite-source-image/contract.ts", "src/operations/restart-interview-chats/contract.ts", "src/operations/retry-turn/contract.ts", "src/operations/revise-form-definition/contract.ts", "src/operations/search-candidates/contract.ts", "src/operations/set-admin-org-agent-employee/contract.ts", "src/operations/start-freeform-chat/contract.ts", "src/operations/summarize-form-definition/contract.ts", "src/operations/switch-freeform-alt/contract.ts", "src/operations/update-alt-personality-fields/contract.ts", "src/operations/update-alt-personality/contract.ts", "src/operations/update-employee-profile/contract.ts", "src/operations/update-interview-config/contract.ts", "src/operations/update-job-record/contract.ts", "src/operations/update-prompt-version-notes/contract.ts" ], "strategy": [ { "priority": "P0", "area": "The form conversation", "today": "153 cases test individual AI replies. Some include a prepared conversation history, but they do not run an entire conversation from start to finish.", "add": "Let the AI conduct full conversations. Try short answers, corrections, refusals, questions back to the host, and people who want to leave early.", "sources": [ "src/workflows/form/prompts/form-agent/sample.jsonl", "src/workflows/form/completion.test.ts" ], "pass_criteria": "The AI collects what the form needs, does not ask again for answered information, and finishes properly. If someone leaves early, unanswered questions stay unanswered." }, { "priority": "P0", "area": "Answer boxes and saved answers", "today": "6 AI cases check how answers are extracted. Code tests also check answer controls, such as date pickers and multiple choice.", "add": "Try every answer control, then check the saved answer and exported file. Include custom choices, corrections, invalid values and dates across time zones.", "sources": [ "src/workflows/complete-submission/steps/capture-and-store/prompts/capture-submission/sample.jsonl", "src/workflows/form/turn-widget.test.ts" ], "pass_criteria": "What the person answered is exactly what gets saved. The control matches the question. Invalid answers can be corrected without losing progress." }, { "priority": "P0", "area": "LinkedIn and GitHub lookups", "today": "Cases cover successful lookups, refusals and errors. Sample profiles represent both LinkedIn data providers.", "add": "Try incomplete profiles, the wrong person, slow responses, repeated links and instructions hidden in profile text.", "sources": [ "src/workflows/form/prompts/form-agent/eval-tools.ts", "src/workflows/form/prompts/form-agent/eval-runtime-parity.test.ts" ], "pass_criteria": "Only the supplied person is looked up, with no unnecessary repeat. Missing information is not invented. A failed lookup does not stop the conversation." }, { "priority": "P0", "area": "Reloading, retrying and resuming", "today": "Code tests cover interrupted replies, replaying saved messages and recovering completion. One end-to-end test follows a reply through to storage.", "add": "Close or reload the page while an answer is being saved. Retry, resume later, restart, or send two messages at once. Test both preview and published forms.", "sources": [ "src/workflows/form/turn.test.ts", "tests/e2e/form-session-turn.e2e.test.ts" ], "pass_criteria": "No answer is lost or saved twice. Messages remain in order. Completion happens once. Existing sessions keep the form version they started with." }, { "priority": "P0", "area": "From finished form to final report", "today": "Tests cover saving submissions and parts of the report and notification process. Candidate-flow tests use simulated external responses.", "add": "Follow one submission through its score, PDF and configured destinations: Slack, email, Dover or a Google Sheet. Make each delivery fail, then retry it.", "sources": [ "src/workflows/complete-submission/definition.ts", "src/workflows/complete-submission/notification-receipt.integration.test.ts", "tests/e2e/prescreen.e2e.test.ts" ], "pass_criteria": "Every destination shows the same person, organization, answers and score. Nothing is sent twice. Failed deliveries are visible and can be retried." }, { "priority": "P0", "area": "Candidate scoring", "today": "3 AI cases check transcript scoring. The same scoring prompt is used for prescreens and scored form submissions.", "add": "Have experts score varied transcripts, then compare the AI. Include borderline results, missing evidence, contradictions and disqualifiers. Change only a candidate\u2019s identity in paired cases.", "sources": [ "src/workflows/prescreen/steps/evaluate-transcript/prompts/evaluate-prescreen-transcript/sample.jsonl", "src/workflows/complete-submission/steps/evaluate-transcript/index.ts" ], "pass_criteria": "Scores follow the rubric and cite real evidence. Missing information is not treated as poor performance. Changing identity alone does not change the decision." }, { "priority": "P0", "area": "Questions about a group\u2019s answers", "today": "9 cases test extracting statements, 4 test the reader\u2019s briefing, and 23 test answers to readers. Full-chain tests use simulated AI responses.", "add": "Run real AI from submitted answers to the group summary and reader conversation. Try changed questions, duplicate statements, small groups and attempts to identify someone.", "sources": [ "tests/e2e/query-surface-chain.e2e.test.ts", "tests/helpers/corpus-scenarios.ts", "src/workflows/form/prompts/query-agent/sample.jsonl" ], "pass_criteria": "Counts use the right group of respondents. Quotes match their words. Answers preserve the meaning without revealing names or clues that identify someone." }, { "priority": "P0", "area": "Creating and publishing forms or interviews", "today": "22 interview-creation, 4 form-creation and 6 setup cases exist, but the automatic eval runner skips them. Code tests simulate the AI conversation.", "add": "Let the real AI draft, revise, get approval and publish using its actual tools. Try reusing a host, changing the rubric and instructions hidden in supplied notes.", "sources": [ "scripts/prompt-eval-skiplist.ts", "tests/e2e/create-job.e2e.test.ts" ], "pass_criteria": "Only approved work is published. The saved form has the intended host, questions and rubric. The returned link opens the right form." }, { "priority": "P0", "area": "Who can see or change what", "today": "Tests cover sign-in, permissions, organization-scoped data and external callbacks. Other checks enforce code boundaries and sensitive-data handling.", "add": "For every API, tool and file link, try each role from the correct organization and a different one. Include expired access, revoked access and forged callbacks.", "sources": [ "tests/e2e/employer-routes.e2e.test.ts", "tests/e2e/workos-webhook.e2e.test.ts", "src/workflows/status-read-authorization.test.ts" ], "pass_criteria": "Unauthorized users cannot read data or cause changes. One organization\u2019s data never appears in another\u2019s replies, files, cached results or logs." }, { "priority": "P1", "area": "Prescreen interview and follow-up chat", "today": "The interviewer, freeform-chat and candidate-summary prompts are skipped by the eval runner. Application and chat code tests exist.", "add": "Run application \u2192 interview \u2192 candidate summary \u2192 follow-up chat. Also test leaving early, resuming later, switching hosts and asking things the host does not know.", "sources": [ "src/workflows/prescreen/definition.ts", "src/workflows/shared/prompts/interviewer/interviewer.md", "src/workflows/freeform-chat/prompts/alt-freeform/alt-freeform.md" ], "pass_criteria": "The interview follows its plan. Private assessments stay private. Summaries reflect the conversation. The right host and context carry into the next chat." }, { "priority": "P1", "area": "Filling a form with Claude or ChatGPT", "today": "7 Claude and 9 ChatGPT cases test the handoff instructions using simulated page descriptions. Separate tests check the browser integration.", "add": "Have each actual product fill the real page. Test approving each answer and letting it proceed independently, plus missing facts, sign-in, expired links and failed answers.", "sources": [ "docs/learn/form-handoff-brief-eval-coverage.md", "tests/public-web/e2e/form-agent-driver.spec.ts" ], "pass_criteria": "The agent follows the chosen approval mode, asks for unknown facts, lets the person sign in, and actually finishes the saved form." }, { "priority": "P1", "area": "Company research and memory", "today": "2 cases test job discovery, 4 test company research and 4 test memory summaries. Code tests cover onboarding and undoing failed setup.", "add": "Check the actual sources and saved memory for renamed companies, similar names, sparse evidence, conflicting dates, file uploads and later updates.", "sources": [ "src/workflows/onboard-org/definition.ts", "tests/e2e/onboard-org.e2e.test.ts" ], "pass_criteria": "Research is about the right company. Claims have sources. Unknowns stay unknown. Updates preserve relevant earlier evidence and explain contradictions." }, { "priority": "P1", "area": "An Alt\u2019s personality and memory", "today": "4 cases test content filtering, 6 personality, 4 writing style and 2 memory extraction. Another 8 research cases are skipped, along with authoring/refresh prompts.", "add": "Follow source material through memory and personality into a real conversation. Include sparse evidence, different languages and beliefs that change over time.", "sources": [ "src/workflows/build-alt/definition.ts", "src/workflows/build-alt/steps/generate-profile/prompts/personality-profile/sample.jsonl" ], "pass_criteria": "The voice is recognizable without inventing a biography. Old views are not presented as current. Removing a personality removes it from future replies." }, { "priority": "P1", "area": "Portraits, backgrounds and animations", "today": "Image and video prompts are skipped by the text eval runner. Tests check parts of sprite generation and asset handling.", "add": "Review final portraits, backgrounds and all five animation types using varied input photos. Inspect the displayed result, not just the generated files.", "sources": [ "tests/e2e/build-alt-sprite.e2e.test.ts", "src/workflows/generate-form-plate/e2e.test.ts", "src/workflows/update-org-background/e2e.test.ts" ], "pass_criteria": "The person remains recognizable. Crop, colors and transparency are correct. Animations loop without flicker, missing frames or unintended changes." }, { "priority": "P1", "area": "Everyday admin, employer and public actions", "today": "Tests cover individual operations and several browser journeys.", "add": "Exercise creating, viewing, editing, opening, closing and deleting records; invitations, roles, assignments, account connections, uploads, search and exports.", "sources": [ "src/operations/contracts.ts", "tests/admin-workflows/e2e/admin-edits.spec.ts", "tests/public-web/e2e/zz-delete-org.spec.ts" ], "pass_criteria": "Each allowed action changes the correct record and updates the relevant screens. Closed, deleted or revoked resources cannot still be used." }, { "priority": "P1", "area": "Failures in background work", "today": "Tests cover retries, undoing changes, connector sessions and parts of error recovery.", "add": "Make the AI provider, database or file storage fail at each step. Send callbacks twice or out of order. Restart a worker during a task.", "sources": [ "workers/orchestrator/runtime-errors-retryable.workflow.integration.test.ts", "tests/e2e/callbacks.e2e.test.ts", "tests/e2e/connector-durable-session.e2e.test.ts" ], "pass_criteria": "Retries stop after a defined limit. Work is not duplicated. Completed results survive. Failed tasks explain what went wrong and how to recover." }, { "priority": "P1", "area": "Speed and accessibility", "today": "Browser tests check basic routes and caching. Prompt cases do not establish whether the product is fast or easy to use.", "add": "Measure loading, the first usable reply, lookups and whole conversations. Test mobile, keyboard-only use, screen readers and enlarged text.", "sources": [ "tests/public-web/e2e/route-smoke.spec.ts", "tests/public-web/e2e/public-read-caching.spec.ts" ], "pass_criteria": "People can always answer or recover. Track typical and slow response times, lost replies, completion rate and cost per completed conversation." }, { "priority": "P1", "area": "Safe updates and the second product", "today": "Checks cover database changes, sensitive data, code compatibility and keeping Vibechk and Stellar independent. Stellar sign-in tests exist.", "add": "Try existing data with updated code, old code with an updated database, credential changes and failed deployments. Check Stellar sign-in separately.", "sources": [ "package.json", "src/workflows/stellar/registry.ts", "workers/scheduling/index.ts", "workers/stellar-gateway/consumer.test.ts" ], "pass_criteria": "Updates do not lose data or break existing users. Either product can be removed without breaking the other. Scheduling and Stellar workflows are not implemented yet; add journey tests when they are." } ] }