Implement self-correcting memory architecture with critic engine

Adds: - Critic Engine (critic.js): Validates output against correction store - Auto-detection: Identifies uncorrected patterns (quantitative without proof, vague quantification, etc.) - Auto-storage: New patterns immediately stored as corrections - NocoDB Schema: Tables for corrections, preferences, episodes, decisions, validation - Memory Service (memory-service.js): NocoDB integration layer - Response Generator (response-generator.js): End-to-end pipeline with critic - Correction Store: JSON-based with README documentation Behavior: - ALL output validated before delivery - Quantitative claims without evidence → auto-corrected - Corrections immediately block future occurrences - System learns from its own mistakes Test: node architecture/test-critic.js
2026-07-04 16:23:05 -07:00
parent 076e9eb605
commit cf3f1e1c53
9 changed files with 1159 additions and 0 deletions
@@ -0,0 +1,329 @@
+#!/usr/bin/env node
+/**
+ * Self-Correcting Memory Architecture - Critic Engine
+ * 
+ * Flow:
+ * 1. Generate response → 2. Critic checks against correction store → 
+ * 3. If violations → Block & regenerate → 4. If new pattern → Auto-store
+ * 
+ * Principles:
+ * - ALL output validated against correction store before delivery
+ * - Quantitative claims without proof → auto-correction
+ * - Corrections immediately block future occurrences
+ * - System learns from its own mistakes
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+class CriticEngine {
+  constructor(options = {}) {
+    this.correctionStorePath = options.correctionStorePath || '/home/jcbeasley/.openclaw/workspace/memory/corrections';
+    this.autoLearnEnabled = options.autoLearn !== false;
+    this.strictMode = options.strictMode || false; // Block on warnings too
+    
+    // Correction patterns (loaded from store + hardcoded rules)
+    this.corrections = [];
+    this.patternRules = this.getPatternRules();
+  }
+
+  /**
+   * Pattern Rules - Auto-detect uncorrected issues
+   */
+  getPatternRules() {
+    return [
+      {
+        id: 'quantitative_without_proof',
+        name: 'Quantitative Claim Without Evidence',
+        pattern: /\b\d+\s*(%|percent|times|x|percent|fold)\b/i,
+        exclude: /\b(according to|based on|from|measured|tested|verified|source:|citation|data:)\b/i,
+        severity: 'auto-correct',
+        message: 'Quantitative claim requires evidence',
+        autoStore: true
+      },
+      {
+        id: 'unverified_performance',
+        name: 'Performance Claim Without Benchmark',
+        pattern: /\b(faster|slower|better|worse|improved|optimized)\s+(than|by)\b/i,
+        exclude: /\b(measured|benchmarked|tested|profiled|verified)\b/i,
+        severity: 'auto-correct',
+        message: 'Performance claims require benchmarks',
+        autoStore: true
+      },
+      {
+        id: 'absolute_without_qualification',
+        name: 'Absolute Statement Without Qualification',
+        pattern: /\b(always|never|all|none|every|impossible)\b/i,
+        exclude: /\b(in this case|for this|under these|given the|based on)\b/i,
+        severity: 'warning',
+        message: 'Absolute statements need qualification',
+        autoStore: false // Manual review first
+      },
+      {
+        id: 'vague_quantification',
+        name: 'Vague Quantification',
+        pattern: /\b(many|few|several|some|most|lots|a lot)\b/i,
+        exclude: /\b(specifically|exactly|precisely)\b/i,
+        severity: 'auto-correct',
+        message: 'Use specific numbers or omit',
+        autoStore: true
+      },
+      {
+        id: 'unverified_completion',
+        name: '"Done" Without Verification',
+        pattern: /\b(done|complete|finished|working)\b/i,
+        exclude: /\b(verified|tested|confirmed|validated|checked)\b/i,
+        severity: 'error',
+        message: 'Completion claims require verification evidence',
+        autoStore: true
+      }
+    ];
+  }
+
+  /**
+   * Load corrections from store
+   */
+  async loadCorrections() {
+    // Load from NocoDB (to be implemented) or local JSON
+    const correctionsFile = path.join(this.correctionStorePath, '_index.json');
+    try {
+      if (fs.existsSync(correctionsFile)) {
+        const data = JSON.parse(fs.readFileSync(correctionsFile, 'utf8'));
+        this.corrections = data.corrections || [];
+      }
+    } catch (err) {
+      console.warn('Could not load corrections:', err.message);
+    }
+    return this;
+  }
+
+  /**
+   * Main Critic Check
+   * Returns: { passed: bool, violations: [], newPatterns: [], corrected: string }
+   */
+  async critique(response, context = {}) {
+    await this.loadCorrections();
+    
+    const violations = [];
+    const newPatterns = [];
+    
+    // 1. Check against known corrections (blocking)
+    const knownViolations = this.checkKnownCorrections(response);
+    violations.push(...knownViolations);
+    
+    // 2. Auto-detect new patterns (if enabled)
+    if (this.autoLearnEnabled && violations.length === 0) {
+      const detectedPatterns = this.detectNewPatterns(response, context);
+      
+      for (const pattern of detectedPatterns) {
+        if (pattern.severity === 'auto-correct') {
+          // Auto-store as new correction
+          await this.storeCorrection(pattern);
+          violations.push({
+            ...pattern,
+            message: `${pattern.message} [AUTO-STORED AS CORRECTION]`,
+            autoStored: true
+          });
+        } else if (pattern.severity === 'warning') {
+          newPatterns.push(pattern);
+        }
+      }
+    }
+    
+    // 3. Generate corrected version if violations found
+    let corrected = response;
+    if (violations.length > 0) {
+      corrected = await this.generateCorrection(response, violations);
+    }
+    
+    return {
+      passed: violations.length === 0,
+      blocked: violations.length > 0 && violations.some(v => v.blocking),
+      violations,
+      newPatterns,
+      corrected,
+      original: response
+    };
+  }
+
+  /**
+   * Check against known corrections (from store)
+   */
+  checkKnownCorrections(response) {
+    const violations = [];
+    
+    for (const correction of this.corrections) {
+      const pattern = new RegExp(correction.pattern, 'i');
+      
+      if (pattern.test(response)) {
+        // Check if exclusion applies
+        if (correction.excludePattern) {
+          const exclude = new RegExp(correction.excludePattern, 'i');
+          if (exclude.test(response)) {
+            continue; // Has exclusion, skip
+          }
+        }
+        
+        violations.push({
+          id: correction.id,
+          type: 'known-correction',
+          severity: correction.severity || 'error',
+          blocking: correction.blocking !== false,
+          message: correction.message,
+          suggestion: correction.suggestion,
+          originalPattern: correction.pattern
+        });
+      }
+    }
+    
+    return violations;
+  }
+
+  /**
+   * Detect new patterns that should become corrections
+   */
+  detectNewPatterns(response, context) {
+    const detected = [];
+    
+    for (const rule of this.patternRules) {
+      if (rule.pattern.test(response)) {
+        // Check if exclusion applies
+        if (rule.exclude && rule.exclude.test(response)) {
+          continue;
+        }
+        
+        detected.push({
+          id: rule.id,
+          type: 'auto-detected',
+          severity: rule.severity,
+          message: rule.message,
+          pattern: rule.pattern.source,
+          autoStore: rule.autoStore,
+          context: context.task || 'unknown'
+        });
+      }
+    }
+    
+    return detected;
+  }
+
+  /**
+   * Store new correction (immediate write)
+   */
+  async storeCorrection(pattern) {
+    const correction = {
+      id: `correction_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+      pattern: pattern.pattern,
+      severity: pattern.severity,
+      message: pattern.message,
+      suggestion: `Add evidence or qualification`,
+      blocking: true,
+      autoDetected: true,
+      createdAt: new Date().toISOString(),
+      context: pattern.context
+    };
+    
+    // Add to memory
+    this.corrections.push(correction);
+    
+    // Persist to disk (immediate)
+    await this.persistCorrections();
+    
+    // Log for audit
+    console.log(`[CRITIC] Auto-stored correction: ${correction.id} - ${pattern.message}`);
+    
+    return correction;
+  }
+
+  /**
+   * Persist corrections to disk (will move to NocoDB)
+   */
+  async persistCorrections() {
+    try {
+      if (!fs.existsSync(this.correctionStorePath)) {
+        fs.mkdirSync(this.correctionStorePath, { recursive: true });
+      }
+      
+      const indexFile = path.join(this.correctionStorePath, '_index.json');
+      fs.writeFileSync(indexFile, JSON.stringify({
+        version: 1,
+        updatedAt: new Date().toISOString(),
+        corrections: this.corrections
+      }, null, 2));
+      
+      // Also write individual correction files for inspection
+      for (const correction of this.corrections.slice(-10)) { // Last 10
+        const filePath = path.join(this.correctionStorePath, `${correction.id}.json`);
+        fs.writeFileSync(filePath, JSON.stringify(correction, null, 2));
+      }
+    } catch (err) {
+      console.error('[CRITIC] Failed to persist corrections:', err);
+    }
+  }
+
+  /**
+   * Generate corrected version of response
+   */
+  async generateCorrection(original, violations) {
+    // Simple correction: add warning header
+    const warnings = violations.map(v => `- ${v.message}`).join('\n');
+    
+    return `⚠️ CORRECTIONS REQUIRED:\n${warnings}\n\n---\n\n${original}`;
+    
+    // Future: Use LLM to actually fix the content
+    // const fixPrompt = buildFixPrompt(original, violations);
+    // return await llm.generate(fixPrompt);
+  }
+
+  /**
+   * Manual correction entry (for user feedback)
+   */
+  async addManualCorrection(originalText, correctionText, reason) {
+    const correction = {
+      id: `manual_${Date.now()}`,
+      pattern: this.escapeRegex(originalText.substring(0, 100)),
+      severity: 'error',
+      message: reason,
+      suggestion: correctionText,
+      blocking: true,
+      manualEntry: true,
+      createdAt: new Date().toISOString()
+    };
+    
+    this.corrections.push(correction);
+    await this.persistCorrections();
+    
+    return correction;
+  }
+
+  /**
+   * Escape regex special chars
+   */
+  escapeRegex(str) {
+    return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  }
+}
+
+// Export
+module.exports = CriticEngine;
+
+// CLI usage
+if (require.main === module) {
+  const responseFile = process.argv[2];
+  if (!responseFile) {
+    console.error('Usage: node critic.js <response-file>');
+    process.exit(1);
+  }
+  
+  const response = fs.readFileSync(responseFile, 'utf8');
+  const critic = new CriticEngine();
+  
+  critic.critique(response)
+    .then(result => {
+      console.log(JSON.stringify(result, null, 2));
+    })
+    .catch(err => {
+      console.error('Critic error:', err);
+      process.exit(1);
+    });
+}