memory

package module
v0.0.0-...-20bcb34 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 18, 2025 License: MIT Imports: 25 Imported by: 0

README

Memory

A high-performance memory and vector database package for Go, built on SQLite with advanced graph and vector search capabilities.

Features

  • SQLite-based: Reliable, embedded database with ACID guarantees
  • Vector Search: Built-in vector similarity search with multiple distance metrics
  • Graph Database: Advanced graph operations with Cypher-like query support
  • Schema Management: Automatic schema creation and migration
  • Embedding Support: Direct integration for AI/ML embeddings
  • High Performance: Optimized for both read and write operations

Extensions

This package includes two powerful SQLite extensions:

SQLite Graph Extension
  • Cypher Query Support: Neo4j-compatible graph queries
  • Graph Algorithms: Shortest path, centrality measures, community detection
  • Performance Optimized: Efficient graph traversal and storage
  • Full Documentation: Comprehensive API reference and tutorials
SQLite Vector Extension
  • Vector Similarity Search: Cosine, Euclidean, and dot product distances
  • Index Optimization: Fast approximate nearest neighbor search
  • Batch Operations: Efficient bulk vector operations
  • Integration Ready: Works seamlessly with embedding models

Installation

go get github.com/agentflare-ai/agentml/memory

Note: This package requires CGO and SQLite development headers.

Quick Start

package main

import (
    "context"
    "database/sql"
    "log"
    
    "github.com/agentflare-ai/agentml/memory"
)

func main() {
    // Open database
    db, err := sql.Open("sqlite3", "memory.db")
    if err != nil {
        log.Fatal(err)
    }
    defer db.Close()
    
    // Load graph extension
    if err := memory.LoadGraphExtension(db); err != nil {
        log.Fatal(err)
    }
    
    // Load vector extension  
    if err := memory.LoadVectorExtension(db); err != nil {
        log.Fatal(err)
    }
    
    // Use the database for graph and vector operations...
}

Graph Operations

-- Create nodes and relationships
CREATE (a:Person {name: 'Alice', age: 30})
CREATE (b:Person {name: 'Bob', age: 25})
CREATE (a)-[:KNOWS]->(b)

-- Query the graph
MATCH (p:Person)-[:KNOWS]->(friend)
WHERE p.age > 25
RETURN p.name, friend.name

Vector Operations

-- Create a vector table
CREATE TABLE embeddings (
    id INTEGER PRIMARY KEY,
    content TEXT,
    vector VECTOR(384)
);

-- Insert vectors
INSERT INTO embeddings (content, vector) 
VALUES ('example text', vector('[0.1, 0.2, 0.3, ...]'));

-- Similarity search
SELECT content, vector_distance(vector, '[0.1, 0.2, ...]') as distance
FROM embeddings
ORDER BY distance
LIMIT 10;

Building Extensions

The package includes build tools for compiling the native extensions:

cd memory
make build-extensions

Performance

  • Graph queries: Optimized for complex traversals
  • Vector search: Sub-millisecond similarity search on large datasets
  • Memory efficient: Minimal overhead over base SQLite
  • Concurrent safe: Full support for concurrent read/write operations

Documentation

License

This project is part of the AgentML ecosystem.

Documentation

Index

Constants

View Source 
const MemoryNamespaceURI = "github.com/agentflare-ai/agentml/memory"

MemoryNamespaceURI is the XML namespace for memory executables.

Variables

View Source 
var (
	// AddNodeSchema defines the schema for adding a node
	AddNodeSchema = &jsonschema.Schema{
		Type:        "object",
		Description: "Add a new node to the graph",
		Properties: map[string]*jsonschema.Schema{
			"labels": {
				Type:        "array",
				Description: "Node labels/types",
				Items:       &jsonschema.Schema{Type: "string"},
			},
			"properties": {
				Type:        "object",
				Description: "Node properties as key-value pairs",
			},
		},
	}

	// AddEdgeSchema defines the schema for adding an edge
	AddEdgeSchema = &jsonschema.Schema{
		Type:        "object",
		Description: "Add a new edge/relationship to the graph",
		Properties: map[string]*jsonschema.Schema{
			"start_node": {
				Type:        "integer",
				Description: "ID of the start node",
			},
			"end_node": {
				Type:        "integer",
				Description: "ID of the end node",
			},
			"type": {
				Type:        "string",
				Description: "Relationship type",
			},
			"properties": {
				Type:        "object",
				Description: "Edge properties as key-value pairs",
			},
		},
		Required: []string{"start_node", "end_node", "type"},
	}

	// UpdateNodeSchema defines the schema for updating a node
	UpdateNodeSchema = &jsonschema.Schema{
		Type:        "object",
		Description: "Update an existing node in the graph",
		Properties: map[string]*jsonschema.Schema{
			"id": {
				Type:        "integer",
				Description: "ID of the node to update",
			},
			"labels": {
				Type:        "array",
				Description: "New labels (replaces existing)",
				Items:       &jsonschema.Schema{Type: "string"},
			},
			"properties": {
				Type:        "object",
				Description: "Properties to update (merges with existing)",
			},
		},
		Required: []string{"id"},
	}

	// UpdateEdgeSchema defines the schema for updating an edge
	UpdateEdgeSchema = &jsonschema.Schema{
		Type:        "object",
		Description: "Update an existing edge in the graph",
		Properties: map[string]*jsonschema.Schema{
			"id": {
				Type:        "integer",
				Description: "ID of the edge to update",
			},
			"type": {
				Type:        "string",
				Description: "New relationship type (optional)",
			},
			"properties": {
				Type:        "object",
				Description: "Properties to update (merges with existing)",
			},
		},
		Required: []string{"id"},
	}

	// GraphUpdateSchema combines all operation schemas
	GraphUpdateSchema = &jsonschema.Schema{
		Type:        "object",
		Description: "A graph update operation",
		AnyOf: []*jsonschema.Schema{
			AddNodeSchema,
			AddEdgeSchema,
			UpdateNodeSchema,
			UpdateEdgeSchema,
		},
	}
)

Graph operation schemas for reuse

View Source 
var (
	// AddVectorSchema defines the schema for adding a vector
	AddVectorSchema = &jsonschema.Schema{
		Type:        "object",
		Description: "Add a new vector to the vector store for similarity search",
		Properties: map[string]*jsonschema.Schema{
			"content": {
				Type:        "string",
				Description: "The text content to embed and index",
			},
			"metadata": {
				Type:        "object",
				Description: "Associated metadata for the vector entry",
				Properties: map[string]*jsonschema.Schema{
					"source": {
						Type:        "string",
						Description: "Source of the content (e.g., 'user_input', 'system_output')",
					},
					"timestamp": {
						Type:        "string",
						Description: "ISO 8601 timestamp when the content was created",
					},
					"context_id": {
						Type:        "string",
						Description: "ID linking this vector to a specific context or conversation",
					},
					"tags": {
						Type:        "array",
						Description: "Tags for categorizing the content",
						Items:       &jsonschema.Schema{Type: "string"},
					},
				},
			},
		},
		Required: []string{"content"},
	}

	// SearchVectorSchema defines the schema for vector similarity search
	SearchVectorSchema = &jsonschema.Schema{
		Type:        "object",
		Description: "Search for similar vectors in the vector store",
		Properties: map[string]*jsonschema.Schema{
			"query": {
				Type:        "string",
				Description: "The text query to search for similar content",
			},
			"limit": {
				Type:        "integer",
				Description: "Maximum number of results to return",
				Minimum:     &[]float64{1}[0],
				Maximum:     &[]float64{100}[0],
			},
			"threshold": {
				Type:        "number",
				Description: "Minimum similarity score threshold (0-1)",
				Minimum:     &[]float64{0}[0],
				Maximum:     &[]float64{1}[0],
			},
		},
		Required: []string{"query"},
	}

	// UpdateVectorSchema defines the schema for updating vector metadata
	UpdateVectorSchema = &jsonschema.Schema{
		Type:        "object",
		Description: "Update metadata for an existing vector",
		Properties: map[string]*jsonschema.Schema{
			"id": {
				Type:        "integer",
				Description: "ID of the vector to update",
			},
			"metadata": {
				Type:        "object",
				Description: "New metadata to merge with existing",
				Properties: map[string]*jsonschema.Schema{
					"source": {
						Type:        "string",
						Description: "Updated source information",
					},
					"tags": {
						Type:        "array",
						Description: "Updated tags",
						Items:       &jsonschema.Schema{Type: "string"},
					},
					"notes": {
						Type:        "string",
						Description: "Additional notes or annotations",
					},
				},
			},
		},
		Required: []string{"id", "metadata"},
	}

	// VectorUpdateSchema combines all vector operation schemas
	VectorUpdateSchema = &jsonschema.Schema{
		Type:        "object",
		Description: "A vector store operation",
		AnyOf: []*jsonschema.Schema{
			AddVectorSchema,
			SearchVectorSchema,
			UpdateVectorSchema,
		},
	}
)

Vector operation schemas for reuse

View Source 
var FunctionCallSchema = &jsonschema.Schema{
	Type: "object",
	Properties: map[string]*jsonschema.Schema{
		"id": {
			Type:        "string",
			Description: "The unique ID of the function call",
		},
		"args": {
			Type:                 "object",
			Description:          "The function parameters and values in JSON object format",
			AdditionalProperties: true,
		},
		"name": {
			Type:        "string",
			Description: "The name of the function to call",
		},
	},
	Required:    []string{"name"},
	Description: "A function call",
}
View Source 
var FunctionResponseSchema = &jsonschema.Schema{
	Type: "object",
	Properties: map[string]*jsonschema.Schema{
		"will_continue": {
			Type:        "boolean",
			Description: "Signals that function call continues, and more responses will be returned, turning the function call into a generator",
		},
		"id": {
			Type:        "string",
			Description: "The ID of the function call this response is for",
		},
		"name": {
			Type:        "string",
			Description: "The name of the function to call",
		},
		"response": {
			Type:                 "object",
			Description:          "The function response in JSON object format",
			AdditionalProperties: true,
		},
	},
	Required:    []string{"name", "response"},
	Description: "A function response",
}
View Source 
var GraphExtension []byte

GraphExtension returns the appropriate extension for the current platform

View Source 
var PartSchema = &jsonschema.Schema{
	Type: "object",
	Properties: map[string]*jsonschema.Schema{
		"text": {
			Type:        "string",
			Description: "The text of the part",
		},
		"thought": {
			Type:        "boolean",
			Description: "Whether the part is thought from the model",
		},
		"thought_signature": {
			Type:        "string",
			Description: "The signature of the thought",
		},
		"function_call":     FunctionCallSchema,
		"function_response": FunctionResponseSchema,
	},
}

PartSchema is a schema for a part of a message genai.Part

View Source 
var VecExtension []byte

VecExtension returns the appropriate extension for the current platform

Functions

func LoadGraphExtension 

func LoadGraphExtension(db *sql.DB) error

LoadGraphExtension loads the embedded graph extension into the database

func LoadVecExtension 

func LoadVecExtension(db *sql.DB) error

LoadVecExtension loads the embedded vector extension into the database

func Loader 

func Loader(deps *Deps) agentml.NamespaceLoader

Loader returns a NamespaceLoader for the memory namespace.

func NewBatchOperationSchema 

func NewBatchOperationSchema(columnsSchema *jsonschema.Schema) *jsonschema.Schema

NewBatchOperationSchema creates a schema for batch operations on a table

func NewDB 

func NewDB(ctx context.Context, dsn string) (db *sql.DB, err error)

func NewDeleteSchema 

func NewDeleteSchema(whereSchema *jsonschema.Schema) *jsonschema.Schema

NewDeleteSchema creates a DELETE schema with the given where conditions

func NewInsertSchema 

func NewInsertSchema(columnsSchema *jsonschema.Schema) *jsonschema.Schema

NewInsertSchema creates an INSERT schema with the given columns

func NewSelectSchema 

func NewSelectSchema(columnsSchema *jsonschema.Schema) *jsonschema.Schema

func NewUpdateSchema 

func NewUpdateSchema(columnsSchema *jsonschema.Schema) *jsonschema.Schema

NewUpdateSchema creates an UPDATE schema with the given columns

func ValidateTableName 

func ValidateTableName(tableName string) error

Types

type BatchOperations 

type BatchOperations struct {
	Inserts []InsertOperation
	Updates []UpdateOperation
	Deletes []DeleteOperation
}

type DB 

type DB = sql.DB

type DBTX 

type DBTX interface {
	ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
	QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
}

DBTX abstracts *sql.DB and *sql.Tx for Exec/Query operations. It matches the subset of methods used by memory executables.

type DeleteOperation 

type DeleteOperation struct {
	Where string
}

type Deps 

type Deps struct {
	DB          *sql.DB
	Graph       *GraphDB
	Vector      *VectorDB
	DefaultDims int
	// Embed computes the embedding for the provided text using the given model.
	Embed func(ctx context.Context, model, text string) ([]float32, error)
	// contains filtered or unexported fields
}

Deps holds dependencies for memory executables.

func InitializeMemorySystem 

func InitializeMemorySystem(ctx context.Context, dsn string, vectorDims int) (*Deps, error)

InitializeMemorySystem creates a fully initialized memory system with DB, Graph, and Vector stores. This is a convenience function that sets up everything needed for memory executables. dsn can be ":memory:" for in-memory database or a file path for persistent storage. vectorDims specifies the dimension for vector embeddings (default: 1536).

type GraphDB 

type GraphDB struct {
	// contains filtered or unexported fields
}

GraphDB represents a knowledge graph

func NewGraphDB 

func NewGraphDB(ctx context.Context, db *sql.DB, tableName string) (*GraphDB, error)

NewGraphDB creates a new graph instance with the given database and table name

func (*GraphDB) Apply 

func (g *GraphDB) Apply(ctx context.Context, update GraphUpdate) error

Apply applies a GraphUpdate operation to the graph

func (*GraphDB) Close 

func (g *GraphDB) Close() error

Close closes the graph (does not close the underlying database connection)

func (*GraphDB) CreateNode 

func (g *GraphDB) CreateNode(ctx context.Context, labels []string, properties map[string]any) (*Node, error)

CreateNode creates a new node with optional labels and properties

func (*GraphDB) CreateRelationship 

func (g *GraphDB) CreateRelationship(ctx context.Context, startNodeID, endNodeID int64, relType string, properties map[string]interface{}) (*Relationship, error)

CreateRelationship creates a relationship between two nodes

func (*GraphDB) DeleteNode 

func (g *GraphDB) DeleteNode(ctx context.Context, nodeID int64) error

func (*GraphDB) FindNodes 

func (g *GraphDB) FindNodes(ctx context.Context, labels []string, properties map[string]any) ([]*Node, error)

FindNodes finds nodes matching the given criteria

func (*GraphDB) FindRelationships 

func (g *GraphDB) FindRelationships(ctx context.Context, relType string, properties map[string]interface{}) ([]*Relationship, error)

FindRelationships finds relationships matching the given criteria

func (*GraphDB) Search 

func (g *GraphDB) Search(ctx context.Context, query string) ([]string, error)

Search performs a search query on the graph and returns matching results as strings

type GraphUpdate 

type GraphUpdate struct {
	Operation  string         `json:"operation"` // "create_node", "create_relationship", "update_node", "update_relationship"
	NodeData   *Node          `json:"node_data,omitempty"`
	RelData    *Relationship  `json:"rel_data,omitempty"`
	Labels     []string       `json:"labels,omitempty"`
	Properties map[string]any `json:"properties,omitempty"`
}

GraphUpdate represents an update operation for the graph

type InsertOperation 

type InsertOperation struct {
	Columns map[string]any `json:"columns"`
}

type Node 

type Node struct {
	ID         int64          `json:"id"`
	Labels     []string       `json:"labels,omitempty"`
	Properties map[string]any `json:"properties,omitempty"`
}

Node represents a node in the graph

type Path 

type Path struct {
	Nodes         []*Node         `json:"nodes"`
	Relationships []*Relationship `json:"relationships"`
}

Path represents a path through the graph

type Relationship 

type Relationship struct {
	ID         int64          `json:"id"`
	StartNode  int64          `json:"start_node"`
	EndNode    int64          `json:"end_node"`
	Type       string         `json:"type"`
	Properties map[string]any `json:"properties,omitempty"`
}

Relationship represents an edge/relationship in the graph

type UpdateOperation 

type UpdateOperation struct {
	Set   []map[string]any `json:"set"`
	Where string           `json:"where"`
}

type VectorDB 

type VectorDB struct {
	// contains filtered or unexported fields
}

VectorDB represents a vector database for embeddings

func NewVectorDB 

func NewVectorDB(ctx context.Context, db *sql.DB, tableName string, dimensions int) (*VectorDB, error)

NewVectorDB creates a new vector database instance with the given database and table name

func (*VectorDB) Close 

func (vs *VectorDB) Close() error

Close closes the vector store (does not close the underlying database connection)

func (*VectorDB) InsertVector 

func (vs *VectorDB) InsertVector(ctx context.Context, id uint64, vector []float32) error

InsertVector inserts a vector with the given ID

func (*VectorDB) SearchSimilarVectors 

func (vs *VectorDB) SearchSimilarVectors(ctx context.Context, queryVector []float32, limit int) ([]VectorResult, error)

SearchSimilarVectors searches for vectors similar to the query vector

type VectorResult 

type VectorResult struct {
	ID       int64     `json:"id"`
	Vector   []float32 `json:"vector"`
	Distance float64   `json:"distance"`
}

VectorResult represents a vector search result

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.