JC Beasley
cf3f1e1c53
Adds: - Critic Engine (critic.js): Validates output against correction store - Auto-detection: Identifies uncorrected patterns (quantitative without proof, vague quantification, etc.) - Auto-storage: New patterns immediately stored as corrections - NocoDB Schema: Tables for corrections, preferences, episodes, decisions, validation - Memory Service (memory-service.js): NocoDB integration layer - Response Generator (response-generator.js): End-to-end pipeline with critic - Correction Store: JSON-based with README documentation Behavior: - ALL output validated before delivery - Quantitative claims without evidence → auto-corrected - Corrections immediately block future occurrences - System learns from its own mistakes Test: node architecture/test-critic.js
107 lines
3.0 KiB
JavaScript
107 lines
3.0 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Test Self-Correcting Memory System
|
|
*
|
|
* Demonstrates:
|
|
* - Auto-detection of uncorrected patterns
|
|
* - Immediate storage to correction store
|
|
* - Blocking of future occurrences
|
|
*/
|
|
|
|
const CriticEngine = require('./critic');
|
|
|
|
// Test cases
|
|
const testCases = [
|
|
{
|
|
name: 'Quantitative claim without evidence',
|
|
input: 'This code is 50% faster than the old implementation.',
|
|
shouldBlock: true,
|
|
shouldAutoStore: true
|
|
},
|
|
{
|
|
name: 'Unverified "done"',
|
|
input: 'The task is done. I finished the deployment.',
|
|
shouldBlock: true,
|
|
shouldAutoStore: true
|
|
},
|
|
{
|
|
name: 'Vague quantification',
|
|
input: 'Many users reported issues with the new feature.',
|
|
shouldBlock: true,
|
|
shouldAutoStore: true
|
|
},
|
|
{
|
|
name: 'Performance claim without benchmark',
|
|
input: 'The new algorithm is 3x faster than the previous one.',
|
|
shouldBlock: true,
|
|
shouldAutoStore: true
|
|
},
|
|
{
|
|
name: 'Good: Quantitative WITH evidence',
|
|
input: 'According to our benchmarks (see test-results.json), the code is 50% faster.',
|
|
shouldBlock: false,
|
|
shouldAutoStore: false
|
|
},
|
|
{
|
|
name: 'Good: Verified completion',
|
|
input: 'The task is complete. Verified by running all tests (100% pass).',
|
|
shouldBlock: false,
|
|
shouldAutoStore: false
|
|
}
|
|
];
|
|
|
|
async function runTests() {
|
|
const critic = new CriticEngine({
|
|
autoLearn: true,
|
|
strictMode: false
|
|
});
|
|
|
|
console.log('=== SELF-CORRECTING MEMORY TEST ===\n');
|
|
|
|
for (const test of testCases) {
|
|
console.log(`Test: ${test.name}`);
|
|
console.log(`Input: "${test.input.substring(0, 60)}..."`);
|
|
|
|
const result = await critic.critique(test.input);
|
|
|
|
console.log(`Result: ${result.passed ? '✅ PASSED' : '❌ BLOCKED'}`);
|
|
console.log(`Violations: ${result.violations.length}`);
|
|
console.log(`Auto-stored: ${result.violations.some(v => v.autoStored) ? 'YES' : 'NO'}`);
|
|
|
|
if (result.violations.length > 0) {
|
|
for (const v of result.violations) {
|
|
console.log(` - ${v.message}${v.autoStored ? ' [AUTO-STORED]' : ''}`);
|
|
}
|
|
}
|
|
|
|
// Verify expectations
|
|
const expectedBlocked = test.shouldBlock;
|
|
const actualBlocked = !result.passed;
|
|
const expectedStored = test.shouldAutoStore;
|
|
const actualStored = result.violations.some(v => v.autoStored);
|
|
|
|
if (expectedBlocked !== actualBlocked) {
|
|
console.log(`⚠️ UNEXPECTED: Expected blocked=${expectedBlocked}, got ${actualBlocked}`);
|
|
}
|
|
if (expectedStored !== actualStored) {
|
|
console.log(`⚠️ UNEXPECTED: Expected stored=${expectedStored}, got ${actualStored}`);
|
|
}
|
|
|
|
console.log('---\n');
|
|
}
|
|
|
|
// Summary
|
|
console.log('\n=== CORRECTION STORE CONTENTS ===');
|
|
console.log(`Total corrections: ${critic.corrections.length}`);
|
|
console.log('Auto-detected patterns:');
|
|
|
|
const autoDetected = critic.corrections.filter(c => c.autoDetected);
|
|
for (const c of autoDetected) {
|
|
console.log(` - ${c.id}: ${c.message}`);
|
|
}
|
|
|
|
console.log('\n=== TEST COMPLETE ===');
|
|
}
|
|
|
|
runTests().catch(console.error);
|