Building a Persistent Memory Agent with Claude API, pgvector, and Redis: A Complete

# docker-compose.yml
version: "3.9"
 
services:
  postgres:
    image: pgvector/pgvector:pg16
    environment:
      POSTGRES_USER: agent_user
      POSTGRES_PASSWORD: your_secure_password
      POSTGRES_DB: agent_memory
    ports:
      - "5432:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/data
      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
 
  redis:
    image: redis:7-alpine
    ports:
      - "6379:6379"
    command: redis-server --appendonly yes
    volumes:
      - redis_data:/data
 
volumes:
  postgres_data:
  redis_data:

-- init.sql
CREATE EXTENSION IF NOT EXISTS vector;
 
-- Long-term memory table
CREATE TABLE long_term_memories (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  user_id TEXT NOT NULL,
  content TEXT NOT NULL,
  content_embedding vector(1536),  -- dimensions for text-embedding-3-small
  memory_type TEXT NOT NULL,       -- 'fact', 'preference', 'skill', 'event'
  importance_score FLOAT DEFAULT 0.5,
  access_count INTEGER DEFAULT 0,
  last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
  created_at TIMESTAMPTZ DEFAULT NOW(),
  expires_at TIMESTAMPTZ           -- NULL = permanent
);
 
-- HNSW index for approximate nearest neighbor search
CREATE INDEX ON long_term_memories
  USING hnsw (content_embedding vector_cosine_ops)
  WITH (m = 16, ef_construction = 64);
 
-- Index for per-user retrieval
CREATE INDEX ON long_term_memories (user_id, created_at DESC);
 
-- Episode memory table (session summaries)
CREATE TABLE episode_memories (
  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
  user_id TEXT NOT NULL,
  session_id TEXT NOT NULL,
  summary TEXT NOT NULL,
  key_topics TEXT[],
  sentiment TEXT,
  created_at TIMESTAMPTZ DEFAULT NOW()
);
 
CREATE INDEX ON episode_memories (user_id, created_at DESC);

// src/memory/LongTermMemory.ts
import { Pool } from "pg";
import OpenAI from "openai";
 
interface Memory {
  id: string;
  content: string;
  memoryType: "fact" | "preference" | "skill" | "event";
  importanceScore: number;
  createdAt: Date;
}
 
export class LongTermMemory {
  private pool: Pool;
  private openai: OpenAI;
 
  constructor() {
    this.pool = new Pool({
      host: process.env.POSTGRES_HOST ?? "localhost",
      port: 5432,
      user: process.env.POSTGRES_USER ?? "agent_user",
      password: process.env.POSTGRES_PASSWORD,
      database: process.env.POSTGRES_DB ?? "agent_memory",
    });
    this.openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
  }
 
  private async embed(text: string): Promise<number[]> {
    const response = await this.openai.embeddings.create({
      model: "text-embedding-3-small",
      input: text,
    });
    return response.data[0].embedding;
  }
 
  async save(
    userId: string,
    content: string,
    type: Memory["memoryType"],
    importanceScore = 0.5
  ): Promise<string> {
    const embedding = await this.embed(content);
    const result = await this.pool.query(
      `INSERT INTO long_term_memories
         (user_id, content, content_embedding, memory_type, importance_score)
       VALUES ($1, $2, $3::vector, $4, $5)
       RETURNING id`,
      [userId, content, JSON.stringify(embedding), type, importanceScore]
    );
    return result.rows[0].id;
  }
 
  async search(
    userId: string,
    query: string,
    limit = 5,
    threshold = 0.75
  ): Promise<Memory[]> {
    const queryEmbedding = await this.embed(query);
 
    const result = await this.pool.query(
      `SELECT id, content, memory_type, importance_score, created_at,
              1 - (content_embedding <=> $2::vector) AS similarity
       FROM long_term_memories
       WHERE user_id = $1
         AND (expires_at IS NULL OR expires_at > NOW())
         AND 1 - (content_embedding <=> $2::vector) > $3
       ORDER BY similarity DESC, importance_score DESC
       LIMIT $4`,
      [userId, JSON.stringify(queryEmbedding), threshold, limit]
    );
 
    // Boost access count so frequently-used memories stay relevant
    if (result.rows.length > 0) {
      const ids = result.rows.map((r) => r.id);
      await this.pool.query(
        `UPDATE long_term_memories
         SET access_count = access_count + 1, last_accessed_at = NOW()
         WHERE id = ANY($1)`,
        [ids]
      );
    }
 
    return result.rows.map((row) => ({
      id: row.id,
      content: row.content,
      memoryType: row.memory_type,
      importanceScore: row.importance_score,
      createdAt: row.created_at,
    }));
  }
 
  async deleteAll(userId: string): Promise<void> {
    await this.pool.query(
      "DELETE FROM long_term_memories WHERE user_id = $1",
      [userId]
    );
    await this.pool.query(
      "DELETE FROM episode_memories WHERE user_id = $1",
      [userId]
    );
  }
}

// src/memory/ShortTermMemory.ts
import { createClient } from "redis";
 
interface Message {
  role: "user" | "assistant";
  content: string;
  timestamp: number;
}
 
export class ShortTermMemory {
  public client: ReturnType<typeof createClient>;
  private ttl: number;
 
  constructor(ttlSeconds = 86400) {
    this.client = createClient({
      url: process.env.REDIS_URL ?? "redis://localhost:6379",
    });
    this.ttl = ttlSeconds;
    this.client.connect();
  }
 
  private key(userId: string, sessionId: string): string {
    return `session:${userId}:${sessionId}`;
  }
 
  async push(
    userId: string,
    sessionId: string,
    message: Omit<Message, "timestamp">
  ): Promise<void> {
    const key = this.key(userId, sessionId);
    const entry: Message = { ...message, timestamp: Date.now() };
 
    await this.client.rPush(key, JSON.stringify(entry));
    await this.client.expire(key, this.ttl);
 
    // Cap at 50 messages to avoid unbounded growth
    const length = await this.client.lLen(key);
    if (length > 50) {
      await this.client.lTrim(key, -50, -1);
    }
  }
 
  async getHistory(
    userId: string,
    sessionId: string,
    limit = 20
  ): Promise<Message[]> {
    const key = this.key(userId, sessionId);
    const items = await this.client.lRange(key, -limit, -1);
    return items.map((item) => JSON.parse(item));
  }
 
  async endSession(userId: string, sessionId: string): Promise<Message[]> {
    const history = await this.getHistory(userId, sessionId, 100);
    await this.client.del(this.key(userId, sessionId));
    return history;
  }
}

// src/agent/MemoryAgent.ts
import Anthropic from "@anthropic-ai/sdk";
import { LongTermMemory } from "../memory/LongTermMemory";
import { ShortTermMemory } from "../memory/ShortTermMemory";
 
const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY });
 
export class MemoryAgent {
  private longTerm: LongTermMemory;
  private shortTerm: ShortTermMemory;
 
  constructor() {
    this.longTerm = new LongTermMemory();
    this.shortTerm = new ShortTermMemory();
  }
 
  private tools: Anthropic.Tool[] = [
    {
      name: "search_memory",
      description:
        "Search the user's stored memories and past conversation history. " +
        "Always call this at the start of a conversation to retrieve relevant context.",
      input_schema: {
        type: "object" as const,
        properties: {
          query: { type: "string", description: "Natural language search query" },
          limit: { type: "number", description: "Max memories to return (default: 3)" },
        },
        required: ["query"],
      },
    },
    {
      name: "save_memory",
      description:
        "Save important information to the user's long-term memory. " +
        "Use this for preferences, facts, skills, and notable events.",
      input_schema: {
        type: "object" as const,
        properties: {
          content: { type: "string", description: "The information to remember" },
          memory_type: {
            type: "string",
            enum: ["fact", "preference", "skill", "event"],
          },
          importance: {
            type: "number",
            description: "Importance score between 0.0 and 1.0",
          },
        },
        required: ["content", "memory_type"],
      },
    },
  ];
 
  async chat(
    userId: string,
    sessionId: string,
    userMessage: string
  ): Promise<string> {
    await this.shortTerm.push(userId, sessionId, {
      role: "user",
      content: userMessage,
    });
 
    const history = await this.shortTerm.getHistory(userId, sessionId, 10);
    const messages: Anthropic.MessageParam[] = history.map((h) => ({
      role: h.role,
      content: h.content,
    }));
 
    while (true) {
      const response = await client.messages.create({
        model: "claude-opus-4-6",
        max_tokens: 2048,
        system: `You are an AI assistant with persistent memory.
At the start of each conversation, use search_memory to retrieve relevant context.
When you learn something important about the user, use save_memory proactively.
This allows you to build a deeper, more personalized relationship over time.`,
        messages,
        tools: this.tools,
      });
 
      if (response.stop_reason === "end_turn") {
        const text = response.content
          .filter((b) => b.type === "text")
          .map((b) => (b as Anthropic.TextBlock).text)
          .join("");
 
        await this.shortTerm.push(userId, sessionId, {
          role: "assistant",
          content: text,
        });
        return text;
      }
 
      if (response.stop_reason === "tool_use") {
        const assistantMessage: Anthropic.MessageParam = {
          role: "assistant",
          content: response.content,
        };
        messages.push(assistantMessage);
 
        const toolResults: Anthropic.ToolResultBlockParam[] = [];
 
        for (const block of response.content) {
          if (block.type !== "tool_use") continue;
          let result = "";
 
          if (block.name === "search_memory") {
            const input = block.input as { query: string; limit?: number };
            const memories = await this.longTerm.search(
              userId,
              input.query,
              input.limit ?? 3
            );
            result =
              memories.length > 0
                ? memories
                    .map(
                      (m) =>
                        `[${m.memoryType}] ${m.content} (score: ${m.importanceScore})`
                    )
                    .join("\n")
                : "No relevant memories found.";
          }
 
          if (block.name === "save_memory") {
            const input = block.input as {
              content: string;
              memory_type: "fact" | "preference" | "skill" | "event";
              importance?: number;
            };
            const id = await this.longTerm.save(
              userId,
              input.content,
              input.memory_type,
              input.importance ?? 0.5
            );
            result = `Memory saved (ID: ${id})`;
          }
 
          toolResults.push({
            type: "tool_result",
            tool_use_id: block.id,
            content: result,
          });
        }
 
        messages.push({ role: "user", content: toolResults });
      }
    }
  }
}

async endSession(userId: string, sessionId: string): Promise<void> {
  const history = await this.shortTerm.endSession(userId, sessionId);
  if (history.length < 3) return; // Skip trivially short sessions
 
  const conversationText = history
    .map((h) => `${h.role}: ${h.content}`)
    .join("\n");
 
  // Use Haiku for cost-efficient summarization
  const summaryResponse = await client.messages.create({
    model: "claude-haiku-4-5-20251001",
    max_tokens: 512,
    messages: [
      {
        role: "user",
        content: `Summarize the following conversation in 2-3 sentences.
Focus on the main topics discussed and any important decisions or preferences expressed.
 
${conversationText}`,
      },
    ],
  });
 
  const summary =
    summaryResponse.content[0].type === "text"
      ? summaryResponse.content[0].text
      : "";
 
  await this.longTerm["pool"].query(
    `INSERT INTO episode_memories (user_id, session_id, summary)
     VALUES ($1, $2, $3)`,
    [userId, sessionId, summary]
  );
}

// Wrong — will throw a type error
await pool.query("INSERT INTO t VALUES ($1)", [embeddingArray]);
 
// Correct — serialize and cast
await pool.query("INSERT INTO t VALUES ($1::vector)", [
  JSON.stringify(embeddingArray),
]);

this.client.on("error", (err) => console.error("Redis error:", err));
await this.client.connect();

// Complete user data deletion (right to erasure)
async deleteUserData(userId: string): Promise<void> {
  await this.longTerm.deleteAll(userId); // Clears both memory tables
  const keys = await this.shortTerm.client.keys(`session:${userId}:*`);
  if (keys.length > 0) {
    await this.shortTerm.client.del(keys);
  }
}

// Simple embedding cache
async embedCached(text: string): Promise<number[]> {
  const cacheKey = `embed:${createHash('md5').update(text).digest('hex')}`;
  const cached = await this.redisClient.get(cacheKey);
  if (cached) return JSON.parse(cached);
 
  const embedding = await this.embed(text);
  // Cache for 2 hours
  await this.redisClient.setEx(cacheKey, 7200, JSON.stringify(embedding));
  return embedding;
}

-- Increase search quality at query time
SET hnsw.ef_search = 100;
 
-- Or set globally in postgresql.conf
-- hnsw.ef_search = 100

-- Delete memories that score below threshold and haven't been accessed in 90 days
DELETE FROM long_term_memories
WHERE importance_score < 0.3
  AND access_count < 2
  AND last_accessed_at < NOW() - INTERVAL '90 days'
  AND expires_at IS NULL;

CREATE TABLE long_term_memories (
  id UUID NOT NULL,
  user_id TEXT NOT NULL,
  -- other columns...
) PARTITION BY HASH (user_id);
 
CREATE TABLE long_term_memories_p0
  PARTITION OF long_term_memories FOR VALUES WITH (MODULUS 4, REMAINDER 0);
-- Repeat for p1, p2, p3