#!/usr/bin/env node /** * Validation Layer - Post-Response Quality Gate * * After LLM responds, validate against: * - Rules followed * - Format compliance * - Workflow adherence * - Safety constraints */ class ResponseValidator { constructor(workflow) { this.workflow = workflow; this.errors = []; this.warnings = []; } /** * Validate a response against all checks */ validate(response) { this.errors = []; this.warnings = []; this.checkRulesFollowed(response); this.checkFormatCompliance(response); this.checkWorkflowAdherence(response); this.checkSafetyConstraints(response); return { passed: this.errors.length === 0, errors: this.errors, warnings: this.warnings, summary: this.buildSummary() }; } /** * Check 1: Rules Followed */ checkRulesFollowed(response) { // Check for prohibited phrases const prohibited = [ { pattern: /it should work/i, msg: "Assumed working without verification" }, { pattern: /probably|maybe|likely/i, msg: "Speculative language detected" }, { pattern: /i think/i, msg: "Unverified assumption" } ]; for (const check of prohibited) { if (check.pattern.test(response)) { this.errors.push({ category: 'rules', message: check.msg, suggestion: 'Verify before stating' }); } } // Check for required phrases (workflow dependent) const required = this.getRequiredPhrases(); for (const phrase of required) { if (!response.includes(phrase)) { this.warnings.push({ category: 'rules', message: `Missing required element: "${phrase}"`, suggestion: `Add "${phrase}" to response` }); } } } /** * Check 2: Format Compliance */ checkFormatCompliance(response) { const checks = [ { pattern: /^##\s+/m, name: 'Section headers', required: true }, { pattern: /\|.*\|.*\|/m, name: 'Tables (optional)', required: false }, { pattern: /^- \[x\]|^- \[ \]/m, name: 'Checkbox lists (optional)', required: false } ]; for (const check of checks) { if (check.required && !check.pattern.test(response)) { this.errors.push({ category: 'format', message: `Missing required format: ${check.name}`, suggestion: `Add ${check.name.toLowerCase()}` }); } } } /** * Check 3: Workflow Adherence */ checkWorkflowAdherence(response) { const workflowChecks = { 'coding': [ { pattern: /## Verification/, msg: 'Missing verification section' }, { pattern: /## Files Modified/, msg: 'Missing files modified section' } ], 'debug': [ { pattern: /## Cause|## Root Cause/, msg: 'Missing cause/root cause' }, { pattern: /## Fix|## Solution/, msg: 'Missing fix section' } ], 'deploy': [ { pattern: /## Pre-Deployment|## Backup/, msg: 'Missing pre-deployment/backup' }, { pattern: /## Verification/, msg: 'Missing verification section' } ], 'audit': [ { pattern: /## Findings/, msg: 'Missing findings section' }, { pattern: /## Recommendations/, msg: 'Missing recommendations' } ] }; const checks = workflowChecks[this.workflow] || []; for (const check of checks) { if (!check.pattern.test(response)) { this.warnings.push({ category: 'workflow', message: check.msg, suggestion: `Follow ${this.workflow} workflow format` }); } } } /** * Check 4: Safety Constraints */ checkSafetyConstraints(response) { const safetyChecks = [ { pattern: /rm -rf|dd if=|mkfs\.\w+|>\s*\/dev\/\w+/, msg: 'Potentially destructive command', severity: 'error' }, { pattern: /ALTER\s+TABLE\s+.*DROP|DELETE\s+FROM\s+\w+\s+WHERE/i, msg: 'Database destructive operation', severity: 'error' }, { pattern: /chmod\s+777|chown\s+-R/i, msg: 'Overly permissive permissions', severity: 'warning' } ]; for (const check of safetyChecks) { if (check.pattern.test(response)) { const entry = { category: 'safety', message: check.msg, suggestion: 'Review for safety' }; if (check.severity === 'error') { this.errors.push(entry); } else { this.warnings.push(entry); } } } } /** * Get required phrases for current workflow */ getRequiredPhrases() { const phrases = { 'coding': ['Verification'], 'debug': ['Cause', 'Fix'], 'deploy': ['Backup', 'Verification'], 'audit': ['Findings'] }; return phrases[this.workflow] || []; } /** * Build validation summary */ buildSummary() { const parts = []; if (this.errors.length === 0 && this.warnings.length === 0) { parts.push('✅ All checks passed'); } else { if (this.errors.length > 0) { parts.push(`❌ ${this.errors.length} error(s)`); } if (this.warnings.length > 0) { parts.push(`⚠️ ${this.warnings.length} warning(s)`); } } return parts.join(', '); } } // Export for use module.exports = ResponseValidator; // CLI usage if (require.main === module) { const workflow = process.argv[2] || 'coding'; const responseFile = process.argv[3]; if (!responseFile) { console.error('Usage: node validator.js '); process.exit(1); } const fs = require('fs'); const response = fs.readFileSync(responseFile, 'utf8'); const validator = new ResponseValidator(workflow); const result = validator.validate(response); console.log(JSON.stringify(result, null, 2)); }