From 13e31b02c5266b3c9d35dc3aa5b256423640990b Mon Sep 17 00:00:00 2001 From: Seekers2001 Date: Tue, 14 Jul 2026 14:35:39 +0800 Subject: [PATCH] Add relevance checks to verify workflow --- docs/commands.md | 12 +++-- docs/editing-changes.md | 2 +- docs/reviewing-changes.md | 3 +- docs/workflows.md | 10 +++- src/core/templates/workflows/verify-change.ts | 54 ++++++++++++++++--- .../templates/skill-templates-parity.test.ts | 21 ++++++-- 6 files changed, 85 insertions(+), 17 deletions(-) diff --git a/docs/commands.md b/docs/commands.md index 6737eb305c..19067b14eb 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -369,7 +369,7 @@ AI: Reading add-dark-mode artifacts... ### `/opsx:verify` -Validate that implementation matches your change artifacts. Checks completeness, correctness, and coherence. +Validate that implementation matches your change artifacts. Checks completeness, correctness, coherence, and relevance. **Syntax:** ``` @@ -382,7 +382,7 @@ Validate that implementation matches your change artifacts. Checks completeness, | `change-name` | No | Which change to verify (inferred from context if not provided) | **What it does:** -- Checks three dimensions of implementation quality +- Checks four dimensions of implementation quality - Searches codebase for implementation evidence - Reports issues categorized as CRITICAL, WARNING, or SUGGESTION - Does not block archive, but surfaces issues @@ -394,6 +394,7 @@ Validate that implementation matches your change artifacts. Checks completeness, | **Completeness** | All tasks done, all requirements implemented, scenarios covered | | **Correctness** | Implementation matches spec intent, edge cases handled | | **Coherence** | Design decisions reflected in code, patterns consistent | +| **Relevance** | Changed code serves the final implementation, without planning residue | **Example:** ``` @@ -416,15 +417,20 @@ AI: Verifying add-dark-mode... ✓ Naming conventions consistent with design.md ⚠ Design mentions "CSS variables" but implementation uses Tailwind classes + RELEVANCE + ✓ New implementation elements map to requirements or necessary infrastructure + ⚠ Unused theme adapter remains from an earlier approach + SUMMARY ───────────────────────────── Critical issues: 0 - Warnings: 2 + Warnings: 3 Ready to archive: Yes (with warnings) Recommendations: 1. Add test for system preference detection 2. Update design.md to reflect Tailwind usage, or refactor to use CSS variables + 3. Remove the unused theme adapter ``` **Tips:** diff --git a/docs/editing-changes.md b/docs/editing-changes.md index e2fc830bdf..2632b0e2d5 100644 --- a/docs/editing-changes.md +++ b/docs/editing-changes.md @@ -43,7 +43,7 @@ Concretely, after some `/opsx:apply` work: - Want to re-examine the plan? Open the artifacts and read them, or run `openspec show ` in your terminal for a consolidated view. - Found something to change? Edit the artifact (or ask the AI to), then continue. -- Want a structured check that the code matches the plan? Run `/opsx:verify` (expanded command). It reports completeness, correctness, and coherence without blocking anything. See [Workflows: Verify](workflows.md#verify-check-your-work). +- Want a structured check that the code matches the plan? Run `/opsx:verify` (expanded command). It reports completeness, correctness, coherence, and relevance without blocking anything. See [Workflows: Verify](workflows.md#verify-check-your-work). There's no "review phase" to return to, because review is something you can do at any point, including after implementation. diff --git a/docs/reviewing-changes.md b/docs/reviewing-changes.md index c99f6bca37..b98cc93629 100644 --- a/docs/reviewing-changes.md +++ b/docs/reviewing-changes.md @@ -97,13 +97,14 @@ Then re-read the part you changed. Re-draft until it's a plan you'd sign your na ## After the code: verify -Once the work is built, `/opsx:verify` is your second review. It re-reads the artifacts and the code and reports mismatches across three dimensions: +Once the work is built, `/opsx:verify` is your second review. It re-reads the artifacts and the code and reports mismatches across four dimensions: | Dimension | What it checks | |-----------|----------------| | **Completeness** | Every task done, every requirement implemented, scenarios covered | | **Correctness** | The implementation matches the spec's intent, edge cases handled | | **Coherence** | Design decisions actually show up in the code | +| **Relevance** | Code introduced by the change still serves the final implementation | ``` You: /opsx:verify diff --git a/docs/workflows.md b/docs/workflows.md index 82f1e1efa2..0c73264d34 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -260,7 +260,7 @@ The recommended completion flow: #### Verify: Check Your Work -`/opsx:verify` validates implementation against your artifacts across three dimensions: +`/opsx:verify` validates implementation against your artifacts across four dimensions: ```text You: /opsx:verify @@ -282,15 +282,20 @@ AI: Verifying add-auth... ✓ Naming conventions consistent with design.md ⚠ Design mentions "event-driven" but implementation uses polling + RELEVANCE + ✓ New implementation elements map to requirements or necessary infrastructure + ⚠ Unused compatibility branch remains from an earlier approach + SUMMARY ───────────────────────────── Critical issues: 0 - Warnings: 2 + Warnings: 3 Ready to archive: Yes (with warnings) Recommendations: 1. Add test for session timeout scenario 2. Consider refactoring to event-driven as designed, or update design.md + 3. Remove the unused compatibility branch ``` **What verify checks:** @@ -300,6 +305,7 @@ AI: Verifying add-auth... | Completeness | All tasks done, all requirements implemented, scenarios covered | | Correctness | Implementation matches spec intent, edge cases handled | | Coherence | Design decisions reflected in code, patterns consistent | +| Relevance | Changed code serves the final implementation, without planning residue | Verify won't block archive, but it surfaces issues you might want to address first. diff --git a/src/core/templates/workflows/verify-change.ts b/src/core/templates/workflows/verify-change.ts index 5fe28aa1f7..74388fcc44 100644 --- a/src/core/templates/workflows/verify-change.ts +++ b/src/core/templates/workflows/verify-change.ts @@ -10,7 +10,7 @@ import { STORE_SELECTION_GUIDANCE } from './store-selection.js'; export function getVerifyChangeSkillTemplate(): SkillTemplate { return { name: 'openspec-verify-change', - description: 'Verify implementation matches change artifacts. Use when the user wants to validate that implementation is complete, correct, and coherent before archiving.', + description: 'Verify implementation matches change artifacts. Use when the user wants to validate that implementation is complete, correct, coherent, and relevant before archiving.', instructions: `Verify that an implementation matches the change artifacts (specs, tasks, design). ${STORE_SELECTION_GUIDANCE} @@ -48,10 +48,11 @@ ${STORE_SELECTION_GUIDANCE} 4. **Initialize verification report structure** - Create a report structure with three dimensions: + Create a report structure with four dimensions: - **Completeness**: Track tasks and spec coverage - **Correctness**: Track requirement implementation and scenario coverage - **Coherence**: Track design adherence and pattern consistency + - **Relevance**: Track semantic residue and unnecessary scope Each dimension can have CRITICAL, WARNING, or SUGGESTION issues. @@ -112,7 +113,22 @@ ${STORE_SELECTION_GUIDANCE} - Add SUGGESTION: "Code pattern deviation:
" - Recommendation: "Consider following project pattern: " -8. **Generate Verification Report** +8. **Verify Relevance** + + **Semantic Residue Audit**: + - Review code introduced or changed for this change, focusing on implementation files identified in the earlier steps + - For each new API, function, type field, state value, helper, branch, or mapping, identify the requirement, task, design decision, or necessary enabling role it serves + - Look for residue from superseded planning assumptions: unused elements, unreachable branches, placeholder or mock code, duplicated mappings, and extension points disconnected from any requirement or task + - Trace references and data flow before flagging an element as residue + - If an element has no traceable purpose: + - Add WARNING: "Planning residue: " + - Recommendation: "Remove or show the requirement/task it serves" + - If the purpose is uncertain: + - Add SUGGESTION: "Possibly irrelevant code: " + - Recommendation: "Confirm whether is required" + - Pre-existing debt is out of scope; keep findings limited to code introduced or changed for this change + +9. **Generate Verification Report** **Summary Scorecard**: \`\`\` @@ -124,6 +140,7 @@ ${STORE_SELECTION_GUIDANCE} | Completeness | X/Y tasks, N reqs| | Correctness | M/N reqs covered | | Coherence | Followed/Issues | + | Relevance | Clean/N residue | \`\`\` **Issues by Priority**: @@ -136,6 +153,7 @@ ${STORE_SELECTION_GUIDANCE} 2. **WARNING** (Should fix): - Spec/design divergences - Missing scenario coverage + - Confirmed planning residue - Each with specific recommendation 3. **SUGGESTION** (Nice to fix): @@ -153,6 +171,7 @@ ${STORE_SELECTION_GUIDANCE} - **Completeness**: Focus on objective checklist items (checkboxes, requirements list) - **Correctness**: Use keyword search, file path analysis, reasonable inference - don't require perfect certainty - **Coherence**: Look for glaring inconsistencies, don't nitpick style +- **Relevance**: Require reference and data-flow evidence that a changed element serves no requirement, task, design decision, or necessary enabling role - **False Positives**: When uncertain, prefer SUGGESTION over WARNING, WARNING over CRITICAL - **Actionability**: Every issue must have a specific recommendation with file/line references where applicable @@ -160,7 +179,8 @@ ${STORE_SELECTION_GUIDANCE} - If only tasks.md exists: verify task completion only, skip spec/design checks - If tasks + specs exist: verify completeness and correctness, skip design -- If full artifacts: verify all three dimensions +- If full artifacts: verify all four dimensions +- If implementation files cannot be identified: skip relevance and explain why - Always note which checks were skipped and why **Output Format** @@ -220,10 +240,11 @@ ${STORE_SELECTION_GUIDANCE} 4. **Initialize verification report structure** - Create a report structure with three dimensions: + Create a report structure with four dimensions: - **Completeness**: Track tasks and spec coverage - **Correctness**: Track requirement implementation and scenario coverage - **Coherence**: Track design adherence and pattern consistency + - **Relevance**: Track semantic residue and unnecessary scope Each dimension can have CRITICAL, WARNING, or SUGGESTION issues. @@ -284,7 +305,22 @@ ${STORE_SELECTION_GUIDANCE} - Add SUGGESTION: "Code pattern deviation:
" - Recommendation: "Consider following project pattern: " -8. **Generate Verification Report** +8. **Verify Relevance** + + **Semantic Residue Audit**: + - Review code introduced or changed for this change, focusing on implementation files identified in the earlier steps + - For each new API, function, type field, state value, helper, branch, or mapping, identify the requirement, task, design decision, or necessary enabling role it serves + - Look for residue from superseded planning assumptions: unused elements, unreachable branches, placeholder or mock code, duplicated mappings, and extension points disconnected from any requirement or task + - Trace references and data flow before flagging an element as residue + - If an element has no traceable purpose: + - Add WARNING: "Planning residue: " + - Recommendation: "Remove or show the requirement/task it serves" + - If the purpose is uncertain: + - Add SUGGESTION: "Possibly irrelevant code: " + - Recommendation: "Confirm whether is required" + - Pre-existing debt is out of scope; keep findings limited to code introduced or changed for this change + +9. **Generate Verification Report** **Summary Scorecard**: \`\`\` @@ -296,6 +332,7 @@ ${STORE_SELECTION_GUIDANCE} | Completeness | X/Y tasks, N reqs| | Correctness | M/N reqs covered | | Coherence | Followed/Issues | + | Relevance | Clean/N residue | \`\`\` **Issues by Priority**: @@ -308,6 +345,7 @@ ${STORE_SELECTION_GUIDANCE} 2. **WARNING** (Should fix): - Spec/design divergences - Missing scenario coverage + - Confirmed planning residue - Each with specific recommendation 3. **SUGGESTION** (Nice to fix): @@ -325,6 +363,7 @@ ${STORE_SELECTION_GUIDANCE} - **Completeness**: Focus on objective checklist items (checkboxes, requirements list) - **Correctness**: Use keyword search, file path analysis, reasonable inference - don't require perfect certainty - **Coherence**: Look for glaring inconsistencies, don't nitpick style +- **Relevance**: Require reference and data-flow evidence that a changed element serves no requirement, task, design decision, or necessary enabling role - **False Positives**: When uncertain, prefer SUGGESTION over WARNING, WARNING over CRITICAL - **Actionability**: Every issue must have a specific recommendation with file/line references where applicable @@ -332,7 +371,8 @@ ${STORE_SELECTION_GUIDANCE} - If only tasks.md exists: verify task completion only, skip spec/design checks - If tasks + specs exist: verify completeness and correctness, skip design -- If full artifacts: verify all three dimensions +- If full artifacts: verify all four dimensions +- If implementation files cannot be identified: skip relevance and explain why - Always note which checks were skipped and why **Output Format** diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index 65202e9e6e..2666c901d1 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -52,11 +52,11 @@ const EXPECTED_FUNCTION_HASHES: Record = { getArchiveChangeSkillTemplate: 'c511a1c943bcfc5f9f3833b8c0ff284b22d34864a08f5f553cec471ee485d38f', getBulkArchiveChangeSkillTemplate: '0f635913757ae3d1609e111f4a8f699443ca47cbaaf8a1b21eb652f7b96a1d13', getOpsxSyncCommandTemplate: '86cf706886d0f18069e2cfa16948b7357028fd348210efb58588c88c416d8622', - getVerifyChangeSkillTemplate: 'd718c79aad649223a73fdb11036c93fb3842ac5a780f4934d50bfa03c9692683', + getVerifyChangeSkillTemplate: '8246c4e94bddf75369926651b4d5420768eeb13b864af041577311119cf88b94', getOpsxArchiveCommandTemplate: '6985bddb310cb45b6b50350bfcebe31bf67146135ca0084c94930920280970a4', getOpsxOnboardCommandTemplate: '0673f34a0f81fd173bcfb8c3ac83e2b1c617f7b7564e24e5298d3bd5665a05a9', getOpsxBulkArchiveCommandTemplate: '9f444fc7b27a5b788077b5e3aa4f61af45aa8c8004ac8d899d204fa362ff89b7', - getOpsxVerifyCommandTemplate: '011509480a20a60342c993906f0f9280c0e9ba5d019d335bdc1ef4d53213a5a8', + getOpsxVerifyCommandTemplate: 'a1d235ae804661eec80504fe7f1e8f65bfe2c3a17ec4aa3c3a83af0d4b14acdf', getOpsxProposeSkillTemplate: '8dfb5e9c719d5ba547aff0d3953c076dca6b33d7223be98cbffc396b8f1e0048', getOpsxProposeCommandTemplate: '7cd569beb32d99cdabd0b49615a8245160a8e152b6ea67a99fc4dd71e3f39f50', getFeedbackSkillTemplate: 'd7d83c5f7fc2b92fe8f4588a5bf2d9cb315e4c73ec19bcd5ef28270906319a0d', @@ -73,7 +73,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-sync-specs': 'f6a1581eb11a30061795c42582db6fa4f5e1f213b4b7cad9f3cbfbe3e9fb2d97', 'openspec-archive-change': '1821aee5a06afd895d59d1e1d16495e484b6087ecf59ec93460d7d5e7851e772', 'openspec-bulk-archive-change': '7b09b04a440809dd7dbf0b1d7b695cbb8c41184d8d104eb32e82d7cdfb476d18', - 'openspec-verify-change': '9a8735eaaa34c278d2193eb32fa736f4b111d1c47e675971c8df40f81d20c8c3', + 'openspec-verify-change': '2bc8cdaa5582621c8df81794557d1efcb234dda29f9ea3f03c831f52e29aac05', 'openspec-onboard': 'b1b6fc9a1b3ff64dafe9b8c39a761ee1bd001b542d47b4e4deaf058e0aa21256', 'openspec-propose': '0cfc9278123d973929cb4da3ea7ac8ae1b6c84b472eed4fb753657b8347eaeb9', 'openspec-update-change': '77ff4d1f1cd08a57649cce1f25e0ebc4f55d6d032dfde5c301d1b479561b72fa', @@ -194,6 +194,21 @@ describe('skill templates split parity', () => { expect(getFeedbackSkillTemplate().instructions).not.toContain('**Store selection:**'); }); + it('checks semantic planning residue in both verify templates', () => { + const verifyTemplates = [ + getVerifyChangeSkillTemplate().instructions, + getOpsxVerifyCommandTemplate().content, + ]; + + for (const content of verifyTemplates) { + expect(content).toContain('Create a report structure with four dimensions'); + expect(content).toContain('**Relevance**: Track semantic residue and unnecessary scope'); + expect(content).toContain('**Verify Relevance**'); + expect(content).toContain('Trace references and data flow before flagging'); + expect(content).toContain('Pre-existing debt is out of scope'); + } + }); + it('generates no workspace-planning residue in any workflow template (4.1)', () => { const allSkills: Array<[string, () => SkillTemplate]> = [ ['openspec-apply-change', getApplyChangeSkillTemplate],