openclaw-workspace-2026/architecture/pipeline.js

#!/usr/bin/env node
/**
 * Context Pipeline - Preprocessing Layer
 * 
 * Every request flows through:
 * 1. Load Rules → 2. Load Preferences → 3. Load Memory → 4. Build Packet
 */

const fs = require('fs');
const path = require('path');

class ContextPipeline {
  constructor() {
    this.rules = [];
    this.preferences = [];
    this.memory = [];
    this.patterns = [];
    this.context = {};
  }

  /**
   * Step 3b: Load Relevant Cross-Project Patterns
   */
  async loadPatterns(taskInput) {
    const manifestPath = path.join('/home/jcbeasley/.openclaw/workspace', 'patterns', 'patterns.json');
    try {
      const manifestRaw = fs.readFileSync(manifestPath, 'utf8');
      const manifest = JSON.parse(manifestRaw);
      const query = (taskInput || '').toLowerCase();

      for (const pattern of manifest.patterns || []) {
        const tokens = [
          ...(pattern.tags || []),
          ...(pattern.affected_projects || []),
          pattern.id,
          pattern.name
        ].map(t => t.toLowerCase());

        const matched = tokens.some(token => query.includes(token));
        if (!matched) continue;

        const patternFile = path.join('/home/jcbeasley/.openclaw/workspace', pattern.file);
        try {
          const content = fs.readFileSync(patternFile, 'utf8');
          this.patterns.push({
            source: pattern.file,
            content: content,
            priority: 6,
            id: pattern.id
          });
        } catch (err) {
          // Pattern file missing; skip
        }
      }
    } catch (err) {
      // Manifest missing or invalid; skip pattern loading
    }

    return this;
  }

  /**
   * Step 1: Load Behavior Rules (always injected)
   */
  async loadRules() {
    const ruleFiles = [
      'SOUL.md',
      'AGENTS.md', 
      'IDENTITY.md',
      'MEMORY.md'
    ];

    for (const file of ruleFiles) {
      const content = await this.readFile(file);
      if (content) {
        this.rules.push({
          source: file,
          content: content,
          priority: 100  // Absolute priority
        });
      }
    }

    // Load active workflow rules if specified
    const workflow = process.env.ACTIVE_WORKFLOW;
    if (workflow) {
      const workflowRules = await this.readFile(`workflows/${workflow}.md`);
      if (workflowRules) {
        this.rules.push({
          source: `workflows/${workflow}.md`,
          content: workflowRules,
          priority: 95
        });
      }
    }

    return this;
  }

  /**
   * Step 2: Load Persistent Preferences (from structured memory)
   */
  async loadPreferences() {
    const prefFiles = [
      'memory/items/mem_pref:communication_style.json',
      'memory/items/mem_pref:format.json',
      'memory/items/mem_pref:memory_persistence.json',
      'memory/items/mem_identity:name.json',
      'memory/items/mem_identity:role.json'
    ];

    for (const file of prefFiles) {
      const data = await this.readJson(file);
      if (data) {
        this.preferences.push({
          source: file,
          content: data.content,
          priority: data.importance || 7,
          tags: data.tags || []
        });
      }
    }

    return this;
  }
  
  /**
   * Step 3a: Load Relevant Memory
   */
  async loadRelevantMemory(query, maxResults = 5) {
    // For now, load recent episodic memories
    // Future: implement vector search
    
    const memoryFiles = [
      'memory/2026-07-04.md',
      'memory/2026-07-03.md'
    ];

    for (const file of memoryFiles) {
      const content = await this.readFile(file);
      if (content) {
        this.memory.push({
          source: file,
          content: content.substring(0, 2000),  // Truncate
          priority: 5
        });
      }
    }

    // Load project memory if PROJECT context set
    const project = process.env.CURRENT_PROJECT;
    if (project) {
      const projectStatus = await this.readFile(`Projects/${project}/memory/STATUS.md`);
      if (projectStatus) {
        this.memory.push({
          source: `Projects/${project}/memory/STATUS.md`,
          content: projectStatus,
          priority: 8
        });
      }
    }

    return this;
  }

  /**
   * Step 4: Build Context Packet
   */
  buildPacket(taskInput) {
    // Sort by priority (highest first)
    const allContext = [
      ...this.rules,
      ...this.preferences.sort((a, b) => b.priority - a.priority),
      ...this.memory,
      ...this.patterns
    ];

    const packet = {
      system: allContext.filter(c => c.priority >= 90).map(c => c.content).join('\n\n---\n\n'),
      preferences: allContext.filter(c => c.priority >= 5 && c.priority < 90).map(c => c.content).join('\n\n'),
      memory: allContext.filter(c => c.priority < 5).map(c => c.content).join('\n\n'),
      task: taskInput,
      metadata: {
        rules_count: this.rules.length,
        prefs_count: this.preferences.length,
        memory_count: this.memory.length,
        patterns_count: this.patterns.length
      }
    };

    return packet;
  }

  /**
   * Utility: Read file from workspace
   */
  async readFile(filepath) {
    const fullPath = path.join('/home/jcbeasley/.openclaw/workspace', filepath);
    try {
      return fs.readFileSync(fullPath, 'utf8');
    } catch (err) {
      return null;
    }
  }

  /**
   * Utility: Read JSON file
   */
  async readJson(filepath) {
    const fullPath = path.join('/home/jcbeasley/.openclaw/workspace', filepath);
    try {
      const content = fs.readFileSync(fullPath, 'utf8');
      return JSON.parse(content);
    } catch (err) {
      return null;
    }
  }
}

// Export for use
module.exports = ContextPipeline;

// CLI usage
if (require.main === module) {
  const pipeline = new ContextPipeline();
  
  pipeline
    .loadRules()
    .then(() => pipeline.loadPreferences())
    .then(() => pipeline.loadRelevantMemory(process.argv[2] || ''))
    .then(() => pipeline.loadPatterns(process.argv[2] || ''))
    .then(() => {
      const packet = pipeline.buildPacket(process.argv[2] || 'No task specified');
      console.log(JSON.stringify(packet, null, 2));
    })
    .catch(err => {
      console.error('Pipeline error:', err);
      process.exit(1);
    });
}