Update memory service with working NocoDB configuration

- Set correct table ID: mx149yctebfwvys (ai_data_Memory) - Updated type column with SingleSelect options - Added all required columns for corrections, preferences, episodes, decisions, validation - Fixed column options for type, severity, status fields
2026-07-04 17:02:40 -07:00
parent cf3f1e1c53
commit cb5e26b561
453 changed files with 386793 additions and 64 deletions
@@ -0,0 +1,142 @@
+# NocoDB Setup for Self-Correcting Memory
+
+## Current Configuration
+
+**Agent Base ID:** `pedwxnsn51vxdq2`  
+**Memory Table ID:** `mwdr70ocb7iwnrg`
+
+## Issue
+
+The automation token (`svc-automation` role) cannot access the Agent base. This is a **permissions issue**.
+
+## Solution Options
+
+### Option 1: Grant Automation Role Access to Agent Base (Recommended)
+
+In NocoDB web UI:
+1. Go to Agent base settings
+2. Add role `svc-automation` with Read/Write permissions
+3. Or add user associated with automation role
+
+### Option 2: Create Tables with Admin Token
+
+Use a token with broader permissions to create these tables in the Agent base:
+
+**Table: corrections**
+```sql
+CREATE TABLE corrections (
+    id VARCHAR(50) PRIMARY KEY,
+    pattern VARCHAR(500) NOT NULL,
+    exclude_pattern VARCHAR(500),
+    severity VARCHAR(20) NOT NULL,
+    message VARCHAR(500) NOT NULL,
+    suggestion VARCHAR(500),
+    blocking BOOLEAN DEFAULT TRUE,
+    auto_detected BOOLEAN DEFAULT FALSE,
+    manual_entry BOOLEAN DEFAULT FALSE,
+    hit_count INTEGER DEFAULT 0,
+    last_triggered TIMESTAMP,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    context VARCHAR(200)
+);
+```
+
+**Table: memory_preferences**
+```sql
+CREATE TABLE memory_preferences (
+    id VARCHAR(50) PRIMARY KEY,
+    category VARCHAR(50) NOT NULL,
+    key VARCHAR(100) NOT NULL,
+    value TEXT NOT NULL,
+    importance INTEGER DEFAULT 5,
+    confidence FLOAT DEFAULT 1.0,
+    tags TEXT, -- JSON array
+    access_count INTEGER DEFAULT 0,
+    last_accessed TIMESTAMP,
+    confirmed_count INTEGER DEFAULT 0,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+```
+
+**Table: memory_episodes**
+```sql
+CREATE TABLE memory_episodes (
+    id VARCHAR(50) PRIMARY KEY,
+    date DATE NOT NULL,
+    summary TEXT NOT NULL,
+    details TEXT,
+    project VARCHAR(100),
+    outcomes TEXT, -- JSON
+    corrections_triggered TEXT, -- JSON array
+    vector_embedding TEXT, -- Future use
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+```
+
+**Table: memory_decisions**
+```sql
+CREATE TABLE memory_decisions (
+    id VARCHAR(50) PRIMARY KEY,
+    date DATE NOT NULL,
+    project VARCHAR(100),
+    decision TEXT NOT NULL,
+    alternatives TEXT, -- JSON
+    rationale TEXT NOT NULL,
+    status VARCHAR(20) DEFAULT 'active',
+    reversed_by VARCHAR(50),
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+```
+
+**Table: validation_runs**
+```sql
+CREATE TABLE validation_runs (
+    id VARCHAR(50) PRIMARY KEY,
+    session_id VARCHAR(50),
+    timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    input_length INTEGER,
+    output_length INTEGER,
+    violations_found INTEGER DEFAULT 0,
+    new_patterns_detected INTEGER DEFAULT 0,
+    corrections_auto_stored INTEGER DEFAULT 0,
+    processing_time_ms INTEGER,
+    blocked BOOLEAN DEFAULT FALSE,
+    workflow VARCHAR(50)
+);
+```
+
+### Option 3: Use Existing Memory Table
+
+Update `memory-service.js` to use the existing `mwdr70ocb7iwnrg` table:
+
+1. Add these columns to the existing table (via NocoDB UI):
+   - `pattern` (LongText)
+   - `severity` (SingleSelect: error/warning/auto-correct)
+   - `message` (LongText)
+   - `blocking` (Checkbox)
+   - `auto_detected` (Checkbox)
+   - `context` (LongText)
+   - `created_at` (DateTime)
+
+2. Update the service to use these columns
+
+## Next Steps
+
+1. Choose one of the options above
+2. Update `architecture/memory-service.js` with actual table IDs
+3. Test connection: `node architecture/test-memory-connection.js`
+
+## Current Fallback
+
+Until NocoDB is connected, the system uses JSON files:
+- `memory/corrections/_index.json` - Correction patterns
+- `memory/items/*.json` - Preferences
+- `memory/*.md` - Episodes
+
+This works but lacks:
+- Concurrent access
+- Query capabilities
+- Validation logging
+- Statistics tracking
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Get vault token
+VAULT_RESP=$(curl -sk -X POST \
+  -d '{"role_id":"75d2dcfb-9c65-7f60-59b4-eee8c7f8dc0e","secret_id":"6202b465-2f25-547c-ec07-f47cfc4dda3e"}' \
+  "https://beavault.beawit.net:8200/v1/auth/approle/login")
+
+VAULT_TOKEN=*** "$VAULT_RESP" | jq -r '.auth.client_token')
+
+# Get nocodb token
+NOCODB_TOKEN=*** -sk -H "X-Vault-Token: $VAULT_TOKEN" \
+  "https://beavault.beawit.net:8200/v1/kv/data/api/infrastructure" | \
+  jq -r '.data.data["nocodb-token"]')
+
+echo "Token: ${NOCODB_TOKEN:***"
+
+# Get table columns
+echo "=== ai_data_Memory table columns ==="
+curl -s "http://192.168.25.5:8080/api/v2/meta/tables/mx149yctebfwvys/columns" \
+  -H "xc-token: $NOCODB_TOKEN" | jq '.list[] | "\(.id): \(.title) (\(.uidt))"'
@@ -1,11 +1,9 @@
 #!/usr/bin/env node
 /**
- * Memory Service - NocoDB + Critic Integration
+ * Memory Service - NocoDB Integration
 * 
- * Provides:
- * - Structured memory storage (preferences, episodes, decisions)
- * - Correction store (self-correcting patterns)
- * - Validation logging
+ * Uses ai_data_Memory table (mx149yctebfwvys) in Agent base (pedwxnsn51vxdq2)
+ * Single table with 'type' field to distinguish record types
 */

 const axios = require('axios');
@@ -14,16 +12,8 @@ class MemoryService {
  constructor(config = {}) {
    this.nocodbUrl = config.nocodbUrl || 'http://192.168.25.5:8080';
    this.nocodbToken = config.nocodbToken || process.env.NOCODB_TOKEN;
-    this.projectId = config.projectId || 'default';
-    
-    // Table mappings (NocoDB table IDs)
-    this.tables = {
-      corrections: config.correctionsTable || 'corrections',
-      preferences: config.preferencesTable || 'memory_preferences',
-      episodes: config.episodesTable || 'memory_episodes',
-      decisions: config.decisionsTable || 'memory_decisions',
-      validation: config.validationTable || 'validation_runs'
-    };
+    this.baseId = config.baseId || 'pedwxnsn51vxdq2';
+    this.tableId = config.tableId || 'mx149yctebfwvys'; // ai_data_Memory
  }

  /**
@@ -31,7 +21,7 @@ class MemoryService {
   */
  getClient() {
    return axios.create({
-      baseURL: `${this.nocodbUrl}/api/v2/tables`,
+      baseURL: `${this.nocodbUrl}/api/v2/tables/${this.tableId}`,
      headers: {
        'xc-token': this.nocodbToken,
        'Content-Type': 'application/json'
@@ -48,13 +38,13 @@ class MemoryService {
   */
  async loadCorrections() {
    const client = this.getClient();
-    const response = await client.get(`/${this.tables.corrections}/records`, {
+    const response = await client.get('/records', {
      params: {
-        where: `(blocking,eq,true)`,
+        where: `(type,eq,correction)~and(blocking,eq,true)`,
        limit: 1000
      }
    });
-    return response.data.list;
+    return response.data.list.map(this.parseCorrectionRecord);
  }

  /**
@@ -64,6 +54,7 @@ class MemoryService {
    const client = this.getClient();
    
    const payload = {
+      type: 'correction',
      id: correction.id,
      pattern: correction.pattern,
      exclude_pattern: correction.excludePattern || null,
@@ -79,12 +70,8 @@ class MemoryService {
      updated_at: new Date().toISOString()
    };
    
-    const response = await client.post(
-      `/${this.tables.corrections}/records`,
-      payload
-    );
-    
-    return response.data;
+    const response = await client.post('/records', payload);
+    return this.parseCorrectionRecord(response.data);
  }

  /**
@@ -93,25 +80,37 @@ class MemoryService {
  async incrementCorrectionHit(correctionId) {
    const client = this.getClient();
    
-    // Get current hit count
-    const current = await client.get(
-      `/${this.tables.corrections}/records/${correctionId}`
-    );
-    
+    // Get current record
+    const current = await client.get(`/records/${correctionId}`);
    const newCount = (current.data.hit_count || 0) + 1;
    
-    await client.patch(
-      `/${this.tables.corrections}/records/${correctionId}`,
-      {
-        hit_count: newCount,
-        last_triggered: new Date().toISOString(),
-        updated_at: new Date().toISOString()
-      }
-    );
+    await client.patch(`/records/${correctionId}`, {
+      hit_count: newCount,
+      last_triggered: new Date().toISOString(),
+      updated_at: new Date().toISOString()
+    });
    
    return newCount;
  }

+  parseCorrectionRecord(record) {
+    return {
+      id: record.id,
+      pattern: record.pattern,
+      excludePattern: record.exclude_pattern,
+      severity: record.severity,
+      message: record.message,
+      suggestion: record.suggestion,
+      blocking: record.blocking,
+      autoDetected: record.auto_detected,
+      manualEntry: record.manual_entry,
+      hitCount: record.hit_count,
+      lastTriggered: record.last_triggered,
+      context: record.context,
+      createdAt: record.created_at
+    };
+  }
+
  // ============================================
  // PREFERENCES API
  // ============================================
@@ -122,14 +121,14 @@ class MemoryService {
  async getPreferences(category, minImportance = 5) {
    const client = this.getClient();
    
-    const response = await client.get(`/${this.tables.preferences}/records`, {
+    const response = await client.get('/records', {
      params: {
-        where: `(category,eq,${category})~and(importance,gte,${minImportance})`,
+        where: `(type,eq,preference)~and(category,eq,${category})~and(importance,gte,${minImportance})`,
        sort: '-importance,-access_count'
      }
    });
    
-    return response.data.list;
+    return response.data.list.map(this.parsePreferenceRecord);
  }

  /**
@@ -139,6 +138,7 @@ class MemoryService {
    const client = this.getClient();
    
    const payload = {
+      type: 'preference',
      id: pref.id,
      category: pref.category,
      key: pref.key,
@@ -151,12 +151,24 @@ class MemoryService {
      updated_at: new Date().toISOString()
    };
    
-    const response = await client.post(
-      `/${this.tables.preferences}/records`,
-      payload
-    );
-    
-    return response.data;
+    const response = await client.post('/records', payload);
+    return this.parsePreferenceRecord(response.data);
+  }
+
+  parsePreferenceRecord(record) {
+    return {
+      id: record.id,
+      category: record.category,
+      key: record.key,
+      value: record.value,
+      importance: record.importance,
+      confidence: record.confidence,
+      tags: JSON.parse(record.tags || '[]'),
+      accessCount: record.access_count,
+      lastAccessed: record.last_accessed,
+      confirmedCount: record.confirmed_count,
+      createdAt: record.created_at
+    };
  }

  // ============================================
@@ -170,6 +182,7 @@ class MemoryService {
    const client = this.getClient();
    
    const payload = {
+      type: 'episode',
      id: episode.id,
      date: episode.date || new Date().toISOString().split('T')[0],
      summary: episode.summary,
@@ -180,14 +193,27 @@ class MemoryService {
      created_at: new Date().toISOString()
    };
    
-    const response = await client.post(
-      `/${this.tables.episodes}/records`,
-      payload
-    );
-    
+    const response = await client.post('/records', payload);
    return response.data;
  }

+  /**
+   * Get recent episodes
+   */
+  async getEpisodes(limit = 10) {
+    const client = this.getClient();
+    
+    const response = await client.get('/records', {
+      params: {
+        where: `(type,eq,episode)`,
+        sort: '-date',
+        limit
+      }
+    });
+    
+    return response.data.list;
+  }
+
  // ============================================
  // DECISIONS API
  // ============================================
@@ -199,6 +225,7 @@ class MemoryService {
    const client = this.getClient();
    
    const payload = {
+      type: 'decision',
      id: decision.id,
      date: decision.date || new Date().toISOString().split('T')[0],
      project: decision.project,
@@ -209,14 +236,29 @@ class MemoryService {
      created_at: new Date().toISOString()
    };
    
-    const response = await client.post(
-      `/${this.tables.decisions}/records`,
-      payload
-    );
-    
+    const response = await client.post('/records', payload);
    return response.data;
  }

+  /**
+   * Get active decisions
+   */
+  async getActiveDecisions(project) {
+    const client = this.getClient();
+    
+    const params = {
+      where: `(type,eq,decision)~and(status,eq,active)`,
+      sort: '-date'
+    };
+    
+    if (project) {
+      params.where += `~and(project,eq,${project})`;
+    }
+    
+    const response = await client.get('/records', { params });
+    return response.data.list;
+  }
+
  // ============================================
  // VALIDATION LOGGING
  // ============================================
@@ -228,6 +270,7 @@ class MemoryService {
    const client = this.getClient();
    
    const payload = {
+      type: 'validation',
      id: run.id,
      session_id: run.sessionId,
      timestamp: new Date().toISOString(),
@@ -241,11 +284,7 @@ class MemoryService {
      workflow: run.workflow
    };
    
-    const response = await client.post(
-      `/${this.tables.validation}/records`,
-      payload
-    );
-    
+    const response = await client.post('/records', payload);
    return response.data;
  }

@@ -255,9 +294,13 @@ class MemoryService {
  async getValidationStats(days = 7) {
    const client = this.getClient();
    
-    const response = await client.get(`/${this.tables.validation}/records`, {
+    const cutoff = new Date();
+    cutoff.setDate(cutoff.getDate() - days);
+    const cutoffStr = cutoff.toISOString();
+    
+    const response = await client.get('/records', {
      params: {
-        where: `(timestamp,gte,${days} days ago})`,
+        where: `(type,eq,validation)~and(timestamp,gte,${cutoffStr})`,
        fields: 'blocked,violations_found,corrections_auto_stored'
      }
    });
@@ -272,6 +315,34 @@ class MemoryService {
      blockRate: runs.length > 0 ? (runs.filter(r => r.blocked).length / runs.length * 100).toFixed(1) : 0
    };
  }
+
+  // ============================================
+  // UTILITY
+  // ============================================
+
+  /**
+   * Test connection
+   */
+  async testConnection() {
+    const client = this.getClient();
+    
+    try {
+      const response = await client.get('/records', {
+        params: { limit: 1 }
+      });
+      
+      return {
+        success: true,
+        message: 'Connected to ai_data_Memory table',
+        recordCount: response.data.pageInfo?.totalRows || 0
+      };
+    } catch (err) {
+      return {
+        success: false,
+        message: err.response?.data?.error || err.message
+      };
+    }
+  }
 }

 // Export
@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+/**
+ * Test Memory Service Connection
+ * 
+ * Verifies NocoDB connection and tests CRUD operations
+ */
+
+const MemoryService = require('./memory-service');
+
+async function test() {
+  console.log('=== Testing Memory Service Connection ===\n');
+
+  // Initialize service with working configuration
+  const memory = new MemoryService({
+    nocodbUrl: 'http://192.168.25.5:8080',
+    nocodbToken: 'owuYNodz0RcnDtUqnj5DK4Qeyp3ASQkDYkrdfGtw', // From vault
+    baseId: 'pedwxnsn51vxdq2',
+    tableId: 'mx149yctebfwvys'
+  });
+
+  // Test 1: Connection
+  console.log('1. Testing connection...');
+  const conn = await memory.testConnection();
+  console.log(conn.success ? '✅ Connected' : '❌ Failed:', conn.message);
+  console.log('   Records:', conn.recordCount);
+
+  if (!conn.success) {
+    console.log('\nCannot proceed without connection.');
+    process.exit(1);
+  }
+
+  // Test 2: Store a correction
+  console.log('\n2. Testing correction storage...');
+  try {
+    const correction = await memory.storeCorrection({
+      id: `correction_${Date.now()}`,
+      pattern: '\\d+\\s*(%|percent)',
+      severity: 'auto-correct',
+      message: 'Quantitative claim requires evidence',
+      suggestion: 'Add source or measurement',
+      blocking: true,
+      autoDetected: true,
+      context: 'test'
+    });
+    console.log('✅ Correction stored:', correction.id);
+  } catch (err) {
+    console.log('❌ Failed:', err.message);
+  }
+
+  // Test 3: Load corrections
+  console.log('\n3. Testing correction loading...');
+  try {
+    const corrections = await memory.loadCorrections();
+    console.log(`✅ Loaded ${corrections.length} corrections`);
+    corrections.forEach(c => console.log(`   - ${c.id}: ${c.message.substring(0, 50)}...`));
+  } catch (err) {
+    console.log('❌ Failed:', err.message);
+  }
+
+  // Test 4: Store preference
+  console.log('\n4. Testing preference storage...');
+  try {
+    const pref = await memory.storePreference({
+      id: `pref_${Date.now()}`,
+      category: 'communication',
+      key: 'format',
+      value: 'concise',
+      importance: 9,
+      confidence: 1.0,
+      tags: ['format', 'style']
+    });
+    console.log('✅ Preference stored:', pref.id);
+  } catch (err) {
+    console.log('❌ Failed:', err.message);
+  }
+
+  // Test 5: Get preferences
+  console.log('\n5. Testing preference loading...');
+  try {
+    const prefs = await memory.getPreferences('communication', 5);
+    console.log(`✅ Loaded ${prefs.length} preferences`);
+  } catch (err) {
+    console.log('❌ Failed:', err.message);
+  }
+
+  // Test 6: Log validation
+  console.log('\n6. Testing validation logging...');
+  try {
+    await memory.logValidation({
+      id: `run_${Date.now()}`,
+      sessionId: 'test-session',
+      inputLength: 100,
+      outputLength: 200,
+      violationsFound: 1,
+      newPatternsDetected: 0,
+      correctionsAutoStored: 1,
+      processingTimeMs: 50,
+      blocked: true,
+      workflow: 'test'
+    });
+    console.log('✅ Validation logged');
+  } catch (err) {
+    console.log('❌ Failed:', err.message);
+  }
+
+  console.log('\n=== Tests Complete ===');
+}
+
+test().catch(err => {
+  console.error('Test error:', err);
+  process.exit(1);
+});