mirror of
https://github.com/symfony/ai.git
synced 2026-03-23 23:42:18 +01:00
[Store] Add SQLite store bridge
This commit is contained in:
committed by
Oskar Stark
parent
a17bb43553
commit
e04f211ce6
@@ -317,6 +317,10 @@ deptrac:
|
||||
collectors:
|
||||
- type: classLike
|
||||
value: Symfony\\AI\\Store\\Bridge\\Redis\\.*
|
||||
- name: SqliteStore
|
||||
collectors:
|
||||
- type: classLike
|
||||
value: Symfony\\AI\\Store\\Bridge\\Sqlite\\.*
|
||||
- name: SupabaseStore
|
||||
collectors:
|
||||
- type: classLike
|
||||
@@ -562,6 +566,9 @@ deptrac:
|
||||
RedisStore:
|
||||
- StoreComponent
|
||||
- PlatformComponent
|
||||
SqliteStore:
|
||||
- StoreComponent
|
||||
- PlatformComponent
|
||||
SupabaseStore:
|
||||
- StoreComponent
|
||||
- PlatformComponent
|
||||
|
||||
@@ -74,6 +74,7 @@ Similarity Search Examples
|
||||
* `Similarity Search with Pinecone (RAG)`_
|
||||
* `Similarity Search with Qdrant (RAG)`_
|
||||
* `Similarity Search with SurrealDB (RAG)`_
|
||||
* `Similarity Search with SQLite (RAG)`_
|
||||
* `Similarity Search with Symfony Cache (RAG)`_
|
||||
* `Similarity Search with Typesense (RAG)`_
|
||||
* `Similarity Search with Vektor (RAG)`_
|
||||
@@ -107,6 +108,7 @@ Supported Stores
|
||||
* `Qdrant`_
|
||||
* `Redis`_
|
||||
* `S3 Vectors`_
|
||||
* `SQLite`_ (requires ``ext-pdo_sqlite``)
|
||||
* `Supabase`_ (requires manual database setup)
|
||||
* `SurrealDB`_
|
||||
* `Symfony Cache`_ (requires ``symfony/cache`` as additional dependency)
|
||||
@@ -250,6 +252,7 @@ This leads to a store implementing two methods::
|
||||
.. _`Similarity Search with Neo4j (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/neo4j.php
|
||||
.. _`Similarity Search with OpenSearch (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/opensearch.php
|
||||
.. _`Similarity Search with Pinecone (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/pinecone.php
|
||||
.. _`Similarity Search with SQLite (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/sqlite.php
|
||||
.. _`Similarity Search with Symfony Cache (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/cache.php
|
||||
.. _`Similarity Search with Qdrant (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/qdrant.php
|
||||
.. _`Similarity Search with SurrealDB (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/surrealdb.php
|
||||
@@ -280,4 +283,5 @@ This leads to a store implementing two methods::
|
||||
.. _`Symfony Cache`: https://symfony.com/doc/current/components/cache.html
|
||||
.. _`Vektor`: https://github.com/centamiv/vektor
|
||||
.. _`Weaviate`: https://weaviate.io/
|
||||
.. _`SQLite`: https://www.sqlite.org/
|
||||
.. _`Supabase`: https://supabase.com/
|
||||
|
||||
113
docs/components/store/sqlite.rst
Normal file
113
docs/components/store/sqlite.rst
Normal file
@@ -0,0 +1,113 @@
|
||||
SQLite Store
|
||||
============
|
||||
|
||||
The SQLite store provides a lightweight persistent vector store without external dependencies.
|
||||
It uses SQLite for data persistence and FTS5 for full-text search capabilities.
|
||||
|
||||
.. note::
|
||||
|
||||
The ``SQLite`` store loads all vectors into PHP memory for distance calculation during vector queries.
|
||||
The dataset must fit within PHP's memory limit.
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
||||
.. code-block:: terminal
|
||||
|
||||
$ composer require symfony/ai-sqlite-store
|
||||
|
||||
Basic Usage
|
||||
-----------
|
||||
|
||||
Using a file-based SQLite database for persistence::
|
||||
|
||||
use Symfony\AI\Store\Bridge\Sqlite\Store;
|
||||
|
||||
$pdo = new \PDO('sqlite:/path/to/vectors.db');
|
||||
$pdo->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION);
|
||||
|
||||
$store = new Store($pdo, 'my_vectors');
|
||||
$store->setup();
|
||||
|
||||
Using an in-memory SQLite database (for testing)::
|
||||
|
||||
$pdo = new \PDO('sqlite::memory:');
|
||||
$pdo->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION);
|
||||
|
||||
$store = new Store($pdo, 'my_vectors');
|
||||
$store->setup();
|
||||
|
||||
Factory Methods
|
||||
---------------
|
||||
|
||||
Create from a PDO connection::
|
||||
|
||||
$store = Store::fromPdo($pdo, 'my_vectors');
|
||||
|
||||
Create from a Doctrine DBAL connection::
|
||||
|
||||
$store = Store::fromDbal($dbalConnection, 'my_vectors');
|
||||
|
||||
Distance Strategies
|
||||
-------------------
|
||||
|
||||
The SQLite store supports different distance calculation strategies::
|
||||
|
||||
use Symfony\AI\Store\Bridge\Sqlite\Store;
|
||||
use Symfony\AI\Store\Distance\DistanceCalculator;
|
||||
use Symfony\AI\Store\Distance\DistanceStrategy;
|
||||
|
||||
$calculator = new DistanceCalculator(DistanceStrategy::COSINE_DISTANCE);
|
||||
$store = new Store($pdo, 'my_vectors', $calculator);
|
||||
|
||||
Available strategies:
|
||||
|
||||
* ``COSINE_DISTANCE`` (default)
|
||||
* ``EUCLIDEAN_DISTANCE``
|
||||
* ``MANHATTAN_DISTANCE``
|
||||
* ``ANGULAR_DISTANCE``
|
||||
* ``CHEBYSHEV_DISTANCE``
|
||||
|
||||
Text Search
|
||||
-----------
|
||||
|
||||
The SQLite store uses FTS5 for full-text search. Documents with ``_text`` metadata
|
||||
are automatically indexed for text search::
|
||||
|
||||
use Symfony\AI\Store\Query\TextQuery;
|
||||
|
||||
$results = $store->query(new TextQuery('artificial intelligence'));
|
||||
|
||||
Hybrid queries combine vector similarity and text search::
|
||||
|
||||
use Symfony\AI\Store\Query\HybridQuery;
|
||||
|
||||
$results = $store->query(
|
||||
new HybridQuery($vector, 'search terms', 0.5)
|
||||
);
|
||||
|
||||
Metadata Filtering
|
||||
------------------
|
||||
|
||||
The SQLite store supports filtering search results based on document metadata using a callable::
|
||||
|
||||
use Symfony\AI\Store\Document\VectorDocument;
|
||||
|
||||
$results = $store->query($vectorQuery, [
|
||||
'filter' => fn(VectorDocument $doc) => $doc->metadata['category'] === 'products',
|
||||
]);
|
||||
|
||||
Query Options
|
||||
-------------
|
||||
|
||||
The SQLite store supports the following query options:
|
||||
|
||||
* ``maxItems`` (int) - Limit the number of results returned
|
||||
* ``filter`` (callable) - Filter documents by metadata before distance calculation
|
||||
|
||||
Example combining both options::
|
||||
|
||||
$results = $store->query($vectorQuery, [
|
||||
'maxItems' => 5,
|
||||
'filter' => fn(VectorDocument $doc) => $doc->metadata['active'] === true,
|
||||
]);
|
||||
63
examples/rag/sqlite.php
Normal file
63
examples/rag/sqlite.php
Normal file
@@ -0,0 +1,63 @@
|
||||
<?php
|
||||
|
||||
/*
|
||||
* This file is part of the Symfony package.
|
||||
*
|
||||
* (c) Fabien Potencier <fabien@symfony.com>
|
||||
*
|
||||
* For the full copyright and license information, please view the LICENSE
|
||||
* file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
use Symfony\AI\Agent\Agent;
|
||||
use Symfony\AI\Agent\Bridge\SimilaritySearch\SimilaritySearch;
|
||||
use Symfony\AI\Agent\Toolbox\AgentProcessor;
|
||||
use Symfony\AI\Agent\Toolbox\Toolbox;
|
||||
use Symfony\AI\Fixtures\Movies;
|
||||
use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory;
|
||||
use Symfony\AI\Platform\Message\Message;
|
||||
use Symfony\AI\Platform\Message\MessageBag;
|
||||
use Symfony\AI\Store\Bridge\Sqlite\Store;
|
||||
use Symfony\AI\Store\Document\Metadata;
|
||||
use Symfony\AI\Store\Document\TextDocument;
|
||||
use Symfony\AI\Store\Document\Vectorizer;
|
||||
use Symfony\AI\Store\Indexer\DocumentIndexer;
|
||||
use Symfony\AI\Store\Indexer\DocumentProcessor;
|
||||
use Symfony\Component\Uid\Uuid;
|
||||
|
||||
require_once dirname(__DIR__).'/bootstrap.php';
|
||||
|
||||
// initialize the store — file-based SQLite for persistence
|
||||
$pdo = new PDO('sqlite:'.__DIR__.'/var/vectors.db');
|
||||
$pdo->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
|
||||
$store = new Store($pdo, 'movies');
|
||||
$store->setup();
|
||||
|
||||
// create embeddings and documents
|
||||
$documents = [];
|
||||
foreach (Movies::all() as $i => $movie) {
|
||||
$documents[] = new TextDocument(
|
||||
id: Uuid::v4(),
|
||||
content: 'Title: '.$movie['title'].\PHP_EOL.'Director: '.$movie['director'].\PHP_EOL.'Description: '.$movie['description'],
|
||||
metadata: new Metadata($movie),
|
||||
);
|
||||
}
|
||||
|
||||
// create embeddings for documents
|
||||
$platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
|
||||
$vectorizer = new Vectorizer($platform, 'text-embedding-3-small', logger());
|
||||
$indexer = new DocumentIndexer(new DocumentProcessor($vectorizer, $store, logger: logger()));
|
||||
$indexer->index($documents);
|
||||
|
||||
$similaritySearch = new SimilaritySearch($vectorizer, $store);
|
||||
$toolbox = new Toolbox([$similaritySearch], logger: logger());
|
||||
$processor = new AgentProcessor($toolbox);
|
||||
$agent = new Agent($platform, 'gpt-5-mini', [$processor], [$processor]);
|
||||
|
||||
$messages = new MessageBag(
|
||||
Message::forSystem('Please answer all user questions only using SimilaritySearch function.'),
|
||||
Message::ofUser('Which movie fits the theme of the mafia?')
|
||||
);
|
||||
$result = $agent->call($messages);
|
||||
|
||||
echo $result->getContent().\PHP_EOL;
|
||||
@@ -97,6 +97,7 @@
|
||||
"ai-typesense-store": "src/store/src/Bridge/Typesense",
|
||||
"ai-vektor-store": "src/store/src/Bridge/Vektor",
|
||||
"ai-weaviate-store": "src/store/src/Bridge/Weaviate",
|
||||
"ai-s3vectors-store": "src/store/src/Bridge/S3Vectors"
|
||||
"ai-s3vectors-store": "src/store/src/Bridge/S3Vectors",
|
||||
"ai-sqlite-store": "src/store/src/Bridge/Sqlite"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,6 +86,7 @@
|
||||
"symfony/ai-redis-store": "^0.6",
|
||||
"symfony/ai-replicate-platform": "^0.6",
|
||||
"symfony/ai-s3vectors-store": "^0.6",
|
||||
"symfony/ai-sqlite-store": "^0.7",
|
||||
"symfony/ai-scaleway-platform": "^0.6",
|
||||
"symfony/ai-session-message-store": "^0.6",
|
||||
"symfony/ai-store": "^0.6",
|
||||
|
||||
@@ -361,6 +361,7 @@ return static function (DefinitionConfigurator $configurator): void {
|
||||
->append($import('store/qdrant'))
|
||||
->append($import('store/redis'))
|
||||
->append($import('store/s3vectors'))
|
||||
->append($import('store/sqlite'))
|
||||
->append($import('store/supabase'))
|
||||
->append($import('store/surrealdb'))
|
||||
->append($import('store/typesense'))
|
||||
|
||||
43
src/ai-bundle/config/store/sqlite.php
Normal file
43
src/ai-bundle/config/store/sqlite.php
Normal file
@@ -0,0 +1,43 @@
|
||||
<?php
|
||||
|
||||
/*
|
||||
* This file is part of the Symfony package.
|
||||
*
|
||||
* (c) Fabien Potencier <fabien@symfony.com>
|
||||
*
|
||||
* For the full copyright and license information, please view the LICENSE
|
||||
* file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace Symfony\Component\Config\Definition\Configurator;
|
||||
|
||||
use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition;
|
||||
|
||||
return (new ArrayNodeDefinition('sqlite'))
|
||||
->useAttributeAsKey('name')
|
||||
->arrayPrototype()
|
||||
->children()
|
||||
->stringNode('dsn')->cannotBeEmpty()->end()
|
||||
->stringNode('connection')->cannotBeEmpty()->end()
|
||||
->stringNode('table_name')->end()
|
||||
->stringNode('strategy')->end()
|
||||
->end()
|
||||
->validate()
|
||||
->ifTrue(static function ($v) {
|
||||
$hasDsn = isset($v['dsn']) && null !== $v['dsn'];
|
||||
$hasConnection = isset($v['connection']) && null !== $v['connection'];
|
||||
|
||||
return $hasDsn && $hasConnection;
|
||||
})
|
||||
->thenInvalid('Cannot use both "dsn" and "connection" for SQLite store. Choose one.')
|
||||
->end()
|
||||
->validate()
|
||||
->ifTrue(static function ($v) {
|
||||
$hasDsn = isset($v['dsn']) && null !== $v['dsn'];
|
||||
$hasConnection = isset($v['connection']) && null !== $v['connection'];
|
||||
|
||||
return !$hasDsn && !$hasConnection;
|
||||
})
|
||||
->thenInvalid('Either "dsn" or "connection" must be configured for SQLite store.')
|
||||
->end()
|
||||
->end();
|
||||
@@ -107,6 +107,7 @@ use Symfony\AI\Store\Bridge\Qdrant\StoreFactory;
|
||||
use Symfony\AI\Store\Bridge\Redis\Distance as RedisDistance;
|
||||
use Symfony\AI\Store\Bridge\Redis\Store as RedisStore;
|
||||
use Symfony\AI\Store\Bridge\S3Vectors\Store as S3VectorsStore;
|
||||
use Symfony\AI\Store\Bridge\Sqlite\Store as SqliteStore;
|
||||
use Symfony\AI\Store\Bridge\Supabase\Store as SupabaseStore;
|
||||
use Symfony\AI\Store\Bridge\SurrealDb\Store as SurrealDbStore;
|
||||
use Symfony\AI\Store\Bridge\Typesense\Store as TypesenseStore;
|
||||
@@ -1857,6 +1858,54 @@ final class AiBundle extends AbstractBundle
|
||||
}
|
||||
}
|
||||
|
||||
if ('sqlite' === $type) {
|
||||
if (!ContainerBuilder::willBeAvailable('symfony/ai-sqlite-store', SqliteStore::class, ['symfony/ai-bundle'])) {
|
||||
throw new RuntimeException('SQLite store configuration requires "symfony/ai-sqlite-store" package. Try running "composer require symfony/ai-sqlite-store".');
|
||||
}
|
||||
|
||||
foreach ($stores as $name => $store) {
|
||||
$distanceCalculatorDefinition = new Definition(DistanceCalculator::class);
|
||||
$distanceCalculatorDefinition->setLazy(true);
|
||||
|
||||
$container->setDefinition('ai.store.distance_calculator.'.$name, $distanceCalculatorDefinition);
|
||||
|
||||
if (\array_key_exists('strategy', $store) && null !== $store['strategy']) {
|
||||
$distanceCalculatorDefinition->setArgument(0, DistanceStrategy::from($store['strategy']));
|
||||
}
|
||||
|
||||
$definition = new Definition(SqliteStore::class);
|
||||
|
||||
if (\array_key_exists('connection', $store) && null !== $store['connection']) {
|
||||
$definition->setFactory([SqliteStore::class, 'fromDbal'])
|
||||
->setArguments([
|
||||
new Reference(\sprintf('doctrine.dbal.%s_connection', $store['connection'])),
|
||||
$store['table_name'] ?? $name,
|
||||
new Reference('ai.store.distance_calculator.'.$name),
|
||||
]);
|
||||
} else {
|
||||
$pdoDefinition = new Definition(\PDO::class, [$store['dsn']]);
|
||||
$pdoDefinition->addMethodCall('setAttribute', [\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION]);
|
||||
$container->setDefinition('ai.store.sqlite.pdo.'.$name, $pdoDefinition);
|
||||
|
||||
$definition->setArguments([
|
||||
new Reference('ai.store.sqlite.pdo.'.$name),
|
||||
$store['table_name'] ?? $name,
|
||||
new Reference('ai.store.distance_calculator.'.$name),
|
||||
]);
|
||||
}
|
||||
|
||||
$definition
|
||||
->setLazy(true)
|
||||
->addTag('proxy', ['interface' => StoreInterface::class])
|
||||
->addTag('proxy', ['interface' => ManagedStoreInterface::class])
|
||||
->addTag('ai.store');
|
||||
|
||||
$container->setDefinition('ai.store.'.$type.'.'.$name, $definition);
|
||||
$container->registerAliasForArgument('ai.store.'.$type.'.'.$name, StoreInterface::class, $name);
|
||||
$container->registerAliasForArgument('ai.store.'.$type.'.'.$name, StoreInterface::class, $type.'_'.$name);
|
||||
}
|
||||
}
|
||||
|
||||
if ('supabase' === $type) {
|
||||
if (!ContainerBuilder::willBeAvailable('symfony/ai-supabase-store', SupabaseStore::class, ['symfony/ai-bundle'])) {
|
||||
throw new RuntimeException('Supabase store configuration requires "symfony/ai-supabase-store" package. Try running "composer require symfony/ai-supabase-store".');
|
||||
|
||||
@@ -68,6 +68,7 @@ use Symfony\AI\Store\Bridge\Qdrant\Store as QdrantStore;
|
||||
use Symfony\AI\Store\Bridge\Qdrant\StoreFactory;
|
||||
use Symfony\AI\Store\Bridge\Redis\Distance as RedisDistance;
|
||||
use Symfony\AI\Store\Bridge\Redis\Store as RedisStore;
|
||||
use Symfony\AI\Store\Bridge\Sqlite\Store as SqliteStore;
|
||||
use Symfony\AI\Store\Bridge\Supabase\Store as SupabaseStore;
|
||||
use Symfony\AI\Store\Bridge\SurrealDb\Store as SurrealDbStore;
|
||||
use Symfony\AI\Store\Bridge\Typesense\Store as TypesenseStore;
|
||||
@@ -3100,6 +3101,106 @@ class AiBundleTest extends TestCase
|
||||
$this->assertTrue($container->hasAlias(StoreInterface::class));
|
||||
}
|
||||
|
||||
public function testSqliteStoreWithDsnCanBeConfigured()
|
||||
{
|
||||
$container = $this->buildContainer([
|
||||
'ai' => [
|
||||
'store' => [
|
||||
'sqlite' => [
|
||||
'my_sqlite_store' => [
|
||||
'dsn' => 'sqlite:/tmp/test.db',
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$this->assertTrue($container->hasDefinition('ai.store.sqlite.my_sqlite_store'));
|
||||
$this->assertTrue($container->hasDefinition('ai.store.distance_calculator.my_sqlite_store'));
|
||||
$this->assertTrue($container->hasDefinition('ai.store.sqlite.pdo.my_sqlite_store'));
|
||||
|
||||
$definition = $container->getDefinition('ai.store.sqlite.my_sqlite_store');
|
||||
$this->assertSame(SqliteStore::class, $definition->getClass());
|
||||
$this->assertTrue($definition->isLazy());
|
||||
$this->assertTrue($definition->hasTag('ai.store'));
|
||||
|
||||
$this->assertTrue($container->hasAlias('.'.StoreInterface::class.' $my_sqlite_store'));
|
||||
$this->assertTrue($container->hasAlias(StoreInterface::class.' $mySqliteStore'));
|
||||
$this->assertTrue($container->hasAlias('.'.StoreInterface::class.' $sqlite_my_sqlite_store'));
|
||||
$this->assertTrue($container->hasAlias(StoreInterface::class.' $sqliteMySqliteStore'));
|
||||
$this->assertTrue($container->hasAlias(StoreInterface::class));
|
||||
}
|
||||
|
||||
public function testSqliteStoreWithConnectionCanBeConfigured()
|
||||
{
|
||||
$container = $this->buildContainer([
|
||||
'ai' => [
|
||||
'store' => [
|
||||
'sqlite' => [
|
||||
'my_sqlite_store' => [
|
||||
'connection' => 'default',
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$this->assertTrue($container->hasDefinition('ai.store.sqlite.my_sqlite_store'));
|
||||
$this->assertTrue($container->hasDefinition('ai.store.distance_calculator.my_sqlite_store'));
|
||||
|
||||
$definition = $container->getDefinition('ai.store.sqlite.my_sqlite_store');
|
||||
$this->assertSame(SqliteStore::class, $definition->getClass());
|
||||
$this->assertSame([SqliteStore::class, 'fromDbal'], $definition->getFactory());
|
||||
$this->assertTrue($definition->isLazy());
|
||||
$this->assertTrue($definition->hasTag('ai.store'));
|
||||
}
|
||||
|
||||
public function testSqliteStoreWithCustomStrategyCanBeConfigured()
|
||||
{
|
||||
$container = $this->buildContainer([
|
||||
'ai' => [
|
||||
'store' => [
|
||||
'sqlite' => [
|
||||
'my_sqlite_store' => [
|
||||
'dsn' => 'sqlite:/tmp/test.db',
|
||||
'strategy' => 'euclidean',
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$this->assertTrue($container->hasDefinition('ai.store.sqlite.my_sqlite_store'));
|
||||
$this->assertTrue($container->hasDefinition('ai.store.distance_calculator.my_sqlite_store'));
|
||||
|
||||
$distanceCalculator = $container->getDefinition('ai.store.distance_calculator.my_sqlite_store');
|
||||
$this->assertSame(DistanceCalculator::class, $distanceCalculator->getClass());
|
||||
$this->assertEquals(DistanceStrategy::from('euclidean'), $distanceCalculator->getArgument(0));
|
||||
}
|
||||
|
||||
public function testSqliteStoreWithCustomTableNameCanBeConfigured()
|
||||
{
|
||||
$container = $this->buildContainer([
|
||||
'ai' => [
|
||||
'store' => [
|
||||
'sqlite' => [
|
||||
'my_sqlite_store' => [
|
||||
'dsn' => 'sqlite:/tmp/test.db',
|
||||
'table_name' => 'custom_vectors',
|
||||
],
|
||||
],
|
||||
],
|
||||
],
|
||||
]);
|
||||
|
||||
$this->assertTrue($container->hasDefinition('ai.store.sqlite.my_sqlite_store'));
|
||||
|
||||
$definition = $container->getDefinition('ai.store.sqlite.my_sqlite_store');
|
||||
$arguments = $definition->getArguments();
|
||||
|
||||
$this->assertSame('custom_vectors', $arguments[1]);
|
||||
}
|
||||
|
||||
public function testSupabaseStoreCanBeConfigured()
|
||||
{
|
||||
$container = $this->buildContainer([
|
||||
|
||||
3
src/store/src/Bridge/Sqlite/.gitattributes
vendored
Normal file
3
src/store/src/Bridge/Sqlite/.gitattributes
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
/Tests export-ignore
|
||||
/phpunit.xml.dist export-ignore
|
||||
/.git* export-ignore
|
||||
8
src/store/src/Bridge/Sqlite/.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
8
src/store/src/Bridge/Sqlite/.github/PULL_REQUEST_TEMPLATE.md
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
Please do not submit any Pull Requests here. They will be closed.
|
||||
---
|
||||
|
||||
Please submit your PR here instead:
|
||||
https://github.com/symfony/ai
|
||||
|
||||
This repository is what we call a "subtree split": a read-only subset of that main repository.
|
||||
We're looking forward to your PR there!
|
||||
20
src/store/src/Bridge/Sqlite/.github/workflows/close-pull-request.yml
vendored
Normal file
20
src/store/src/Bridge/Sqlite/.github/workflows/close-pull-request.yml
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
name: Close Pull Request
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened]
|
||||
|
||||
jobs:
|
||||
run:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: superbrothers/close-pull-request@v3
|
||||
with:
|
||||
comment: |
|
||||
Thanks for your Pull Request! We love contributions.
|
||||
|
||||
However, you should instead open your PR on the main repository:
|
||||
https://github.com/symfony/ai
|
||||
|
||||
This repository is what we call a "subtree split": a read-only subset of that main repository.
|
||||
We're looking forward to your PR there!
|
||||
4
src/store/src/Bridge/Sqlite/.gitignore
vendored
Normal file
4
src/store/src/Bridge/Sqlite/.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
vendor/
|
||||
composer.lock
|
||||
phpunit.xml
|
||||
.phpunit.result.cache
|
||||
7
src/store/src/Bridge/Sqlite/CHANGELOG.md
Normal file
7
src/store/src/Bridge/Sqlite/CHANGELOG.md
Normal file
@@ -0,0 +1,7 @@
|
||||
CHANGELOG
|
||||
=========
|
||||
|
||||
0.7
|
||||
---
|
||||
|
||||
* Add the bridge
|
||||
19
src/store/src/Bridge/Sqlite/LICENSE
Normal file
19
src/store/src/Bridge/Sqlite/LICENSE
Normal file
@@ -0,0 +1,19 @@
|
||||
Copyright (c) 2026-present Fabien Potencier
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is furnished
|
||||
to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
14
src/store/src/Bridge/Sqlite/README.md
Normal file
14
src/store/src/Bridge/Sqlite/README.md
Normal file
@@ -0,0 +1,14 @@
|
||||
SQLite Store
|
||||
============
|
||||
|
||||
Provides [SQLite](https://www.sqlite.org/) vector store integration for Symfony AI Store.
|
||||
|
||||
Uses SQLite FTS5 for full-text search and PHP-side distance calculation for vector similarity.
|
||||
|
||||
Resources
|
||||
---------
|
||||
|
||||
* [Contributing](https://symfony.com/doc/current/contributing/index.html)
|
||||
* [Report issues](https://github.com/symfony/ai/issues) and
|
||||
[send Pull Requests](https://github.com/symfony/ai/pulls)
|
||||
in the [main Symfony AI repository](https://github.com/symfony/ai)
|
||||
383
src/store/src/Bridge/Sqlite/Store.php
Normal file
383
src/store/src/Bridge/Sqlite/Store.php
Normal file
@@ -0,0 +1,383 @@
|
||||
<?php
|
||||
|
||||
/*
|
||||
* This file is part of the Symfony package.
|
||||
*
|
||||
* (c) Fabien Potencier <fabien@symfony.com>
|
||||
*
|
||||
* For the full copyright and license information, please view the LICENSE
|
||||
* file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace Symfony\AI\Store\Bridge\Sqlite;
|
||||
|
||||
use Doctrine\DBAL\Connection;
|
||||
use Symfony\AI\Platform\Vector\Vector;
|
||||
use Symfony\AI\Store\Distance\DistanceCalculator;
|
||||
use Symfony\AI\Store\Document\Metadata;
|
||||
use Symfony\AI\Store\Document\VectorDocument;
|
||||
use Symfony\AI\Store\Exception\InvalidArgumentException;
|
||||
use Symfony\AI\Store\Exception\UnsupportedQueryTypeException;
|
||||
use Symfony\AI\Store\ManagedStoreInterface;
|
||||
use Symfony\AI\Store\Query\HybridQuery;
|
||||
use Symfony\AI\Store\Query\QueryInterface;
|
||||
use Symfony\AI\Store\Query\TextQuery;
|
||||
use Symfony\AI\Store\Query\VectorQuery;
|
||||
use Symfony\AI\Store\StoreInterface;
|
||||
|
||||
/**
|
||||
* @author Johannes Wachter <johannes@sulu.io>
|
||||
*/
|
||||
final class Store implements ManagedStoreInterface, StoreInterface
|
||||
{
|
||||
public function __construct(
|
||||
private readonly \PDO $connection,
|
||||
private readonly string $tableName,
|
||||
private readonly DistanceCalculator $distanceCalculator = new DistanceCalculator(),
|
||||
) {
|
||||
}
|
||||
|
||||
public function setup(array $options = []): void
|
||||
{
|
||||
if ([] !== $options) {
|
||||
throw new InvalidArgumentException('No supported options.');
|
||||
}
|
||||
|
||||
$this->connection->exec(\sprintf(
|
||||
'CREATE TABLE IF NOT EXISTS %s (id TEXT PRIMARY KEY, vector TEXT NOT NULL, metadata TEXT)',
|
||||
$this->tableName,
|
||||
));
|
||||
|
||||
$this->connection->exec(\sprintf(
|
||||
'CREATE VIRTUAL TABLE IF NOT EXISTS %s_fts USING fts5(id UNINDEXED, content)',
|
||||
$this->tableName,
|
||||
));
|
||||
}
|
||||
|
||||
public static function fromPdo(\PDO $connection, string $tableName, DistanceCalculator $distanceCalculator = new DistanceCalculator()): self
|
||||
{
|
||||
return new self($connection, $tableName, $distanceCalculator);
|
||||
}
|
||||
|
||||
/**
|
||||
* @throws InvalidArgumentException When DBAL connection doesn't use PDO driver
|
||||
*/
|
||||
public static function fromDbal(Connection $connection, string $tableName, DistanceCalculator $distanceCalculator = new DistanceCalculator()): self
|
||||
{
|
||||
$pdo = $connection->getNativeConnection();
|
||||
|
||||
if (!$pdo instanceof \PDO) {
|
||||
throw new InvalidArgumentException('Only DBAL connections using PDO driver are supported.');
|
||||
}
|
||||
|
||||
return self::fromPdo($pdo, $tableName, $distanceCalculator);
|
||||
}
|
||||
|
||||
public function drop(array $options = []): void
|
||||
{
|
||||
$this->connection->exec(\sprintf('DROP TABLE IF EXISTS %s_fts', $this->tableName));
|
||||
$this->connection->exec(\sprintf('DROP TABLE IF EXISTS %s', $this->tableName));
|
||||
}
|
||||
|
||||
public function add(VectorDocument|array $documents): void
|
||||
{
|
||||
if ($documents instanceof VectorDocument) {
|
||||
$documents = [$documents];
|
||||
}
|
||||
|
||||
$this->connection->beginTransaction();
|
||||
|
||||
try {
|
||||
$statement = $this->connection->prepare(\sprintf(
|
||||
'INSERT OR REPLACE INTO %s (id, vector, metadata) VALUES (:id, :vector, :metadata)',
|
||||
$this->tableName,
|
||||
));
|
||||
|
||||
$ftsDeleteStatement = $this->connection->prepare(\sprintf(
|
||||
'DELETE FROM %s_fts WHERE id = :id',
|
||||
$this->tableName,
|
||||
));
|
||||
|
||||
$ftsInsertStatement = $this->connection->prepare(\sprintf(
|
||||
'INSERT INTO %s_fts (id, content) VALUES (:id, :content)',
|
||||
$this->tableName,
|
||||
));
|
||||
|
||||
foreach ($documents as $document) {
|
||||
$id = (string) $document->getId();
|
||||
$metadata = $document->getMetadata()->getArrayCopy();
|
||||
|
||||
$statement->bindValue(':id', $id);
|
||||
$statement->bindValue(':vector', json_encode($document->getVector()->getData()));
|
||||
$statement->bindValue(':metadata', json_encode($metadata));
|
||||
$statement->execute();
|
||||
|
||||
$ftsDeleteStatement->bindValue(':id', $id);
|
||||
$ftsDeleteStatement->execute();
|
||||
|
||||
$text = $document->getMetadata()->getText();
|
||||
if (null !== $text && '' !== $text) {
|
||||
$ftsInsertStatement->bindValue(':id', $id);
|
||||
$ftsInsertStatement->bindValue(':content', $text);
|
||||
$ftsInsertStatement->execute();
|
||||
}
|
||||
}
|
||||
|
||||
$this->connection->commit();
|
||||
} catch (\Throwable $e) {
|
||||
$this->connection->rollBack();
|
||||
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
public function remove(string|array $ids, array $options = []): void
|
||||
{
|
||||
if (\is_string($ids)) {
|
||||
$ids = [$ids];
|
||||
}
|
||||
|
||||
if ([] === $ids) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->connection->beginTransaction();
|
||||
|
||||
try {
|
||||
$placeholders = implode(', ', array_fill(0, \count($ids), '?'));
|
||||
|
||||
$statement = $this->connection->prepare(\sprintf(
|
||||
'DELETE FROM %s WHERE id IN (%s)',
|
||||
$this->tableName,
|
||||
$placeholders,
|
||||
));
|
||||
|
||||
$ftsStatement = $this->connection->prepare(\sprintf(
|
||||
'DELETE FROM %s_fts WHERE id IN (%s)',
|
||||
$this->tableName,
|
||||
$placeholders,
|
||||
));
|
||||
|
||||
foreach ($ids as $index => $id) {
|
||||
$statement->bindValue($index + 1, $id);
|
||||
$ftsStatement->bindValue($index + 1, $id);
|
||||
}
|
||||
|
||||
$statement->execute();
|
||||
$ftsStatement->execute();
|
||||
|
||||
$this->connection->commit();
|
||||
} catch (\Throwable $e) {
|
||||
$this->connection->rollBack();
|
||||
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
public function supports(string $queryClass): bool
|
||||
{
|
||||
return \in_array($queryClass, [
|
||||
VectorQuery::class,
|
||||
TextQuery::class,
|
||||
HybridQuery::class,
|
||||
], true);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{
|
||||
* maxItems?: positive-int,
|
||||
* filter?: callable(VectorDocument): bool,
|
||||
* } $options
|
||||
*/
|
||||
public function query(QueryInterface $query, array $options = []): iterable
|
||||
{
|
||||
return match (true) {
|
||||
$query instanceof VectorQuery => $this->queryVector($query, $options),
|
||||
$query instanceof TextQuery => $this->queryText($query, $options),
|
||||
$query instanceof HybridQuery => $this->queryHybrid($query, $options),
|
||||
default => throw new UnsupportedQueryTypeException($query::class, $this),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{
|
||||
* maxItems?: positive-int,
|
||||
* filter?: callable(VectorDocument): bool,
|
||||
* } $options
|
||||
*
|
||||
* @return iterable<VectorDocument>
|
||||
*/
|
||||
private function queryVector(VectorQuery $query, array $options): iterable
|
||||
{
|
||||
$documents = $this->loadAllDocuments();
|
||||
|
||||
if ([] === $documents) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (isset($options['filter'])) {
|
||||
$documents = array_values(array_filter($documents, $options['filter']));
|
||||
}
|
||||
|
||||
yield from $this->distanceCalculator->calculate($documents, $query->getVector(), $options['maxItems'] ?? null);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{
|
||||
* maxItems?: positive-int,
|
||||
* filter?: callable(VectorDocument): bool,
|
||||
* } $options
|
||||
*
|
||||
* @return iterable<VectorDocument>
|
||||
*/
|
||||
private function queryText(TextQuery $query, array $options): iterable
|
||||
{
|
||||
$searchTerms = $query->getTexts();
|
||||
$ftsQuery = implode(' OR ', array_map(static fn (string $term): string => '"'.$term.'"', $searchTerms));
|
||||
|
||||
$statement = $this->connection->prepare(\sprintf(
|
||||
'SELECT id, rank FROM %s_fts WHERE %s_fts MATCH :query ORDER BY rank',
|
||||
$this->tableName,
|
||||
$this->tableName,
|
||||
));
|
||||
$statement->bindValue(':query', $ftsQuery);
|
||||
$statement->execute();
|
||||
|
||||
$ftsResults = $statement->fetchAll(\PDO::FETCH_ASSOC);
|
||||
|
||||
if ([] === $ftsResults) {
|
||||
return;
|
||||
}
|
||||
|
||||
$matchedIds = array_column($ftsResults, 'id');
|
||||
$placeholders = implode(', ', array_fill(0, \count($matchedIds), '?'));
|
||||
|
||||
$docStatement = $this->connection->prepare(\sprintf(
|
||||
'SELECT id, vector, metadata FROM %s WHERE id IN (%s)',
|
||||
$this->tableName,
|
||||
$placeholders,
|
||||
));
|
||||
|
||||
foreach ($matchedIds as $index => $id) {
|
||||
$docStatement->bindValue($index + 1, $id);
|
||||
}
|
||||
|
||||
$docStatement->execute();
|
||||
|
||||
$documents = [];
|
||||
foreach ($docStatement->fetchAll(\PDO::FETCH_ASSOC) as $row) {
|
||||
$documents[$row['id']] = new VectorDocument(
|
||||
id: $row['id'],
|
||||
vector: new Vector(json_decode($row['vector'], true)),
|
||||
metadata: new Metadata(json_decode($row['metadata'] ?? '{}', true)),
|
||||
);
|
||||
}
|
||||
|
||||
// Preserve FTS rank ordering
|
||||
$orderedDocuments = [];
|
||||
foreach ($matchedIds as $id) {
|
||||
if (isset($documents[$id])) {
|
||||
$orderedDocuments[] = $documents[$id];
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($options['filter'])) {
|
||||
$orderedDocuments = array_values(array_filter($orderedDocuments, $options['filter']));
|
||||
}
|
||||
|
||||
$maxItems = $options['maxItems'] ?? null;
|
||||
$count = 0;
|
||||
|
||||
foreach ($orderedDocuments as $document) {
|
||||
if (null !== $maxItems && $count >= $maxItems) {
|
||||
break;
|
||||
}
|
||||
|
||||
yield $document;
|
||||
++$count;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array{
|
||||
* maxItems?: positive-int,
|
||||
* filter?: callable(VectorDocument): bool,
|
||||
* } $options
|
||||
*
|
||||
* @return iterable<VectorDocument>
|
||||
*/
|
||||
private function queryHybrid(HybridQuery $query, array $options): iterable
|
||||
{
|
||||
$vectorResults = iterator_to_array($this->queryVector(
|
||||
new VectorQuery($query->getVector()),
|
||||
$options,
|
||||
));
|
||||
|
||||
$textResults = iterator_to_array($this->queryText(
|
||||
new TextQuery($query->getTexts()),
|
||||
$options,
|
||||
));
|
||||
|
||||
$mergedResults = [];
|
||||
$seenIds = [];
|
||||
|
||||
foreach ($vectorResults as $doc) {
|
||||
$id = (string) $doc->getId();
|
||||
if (!isset($seenIds[$id])) {
|
||||
$mergedResults[] = new VectorDocument(
|
||||
id: $doc->getId(),
|
||||
vector: $doc->getVector(),
|
||||
metadata: $doc->getMetadata(),
|
||||
score: null !== $doc->getScore() ? $doc->getScore() * $query->getSemanticRatio() : null,
|
||||
);
|
||||
$seenIds[$id] = true;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($textResults as $doc) {
|
||||
$id = (string) $doc->getId();
|
||||
if (!isset($seenIds[$id])) {
|
||||
$mergedResults[] = $doc;
|
||||
$seenIds[$id] = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (isset($options['filter'])) {
|
||||
$mergedResults = array_values(array_filter($mergedResults, $options['filter']));
|
||||
}
|
||||
|
||||
$maxItems = $options['maxItems'] ?? null;
|
||||
$count = 0;
|
||||
|
||||
foreach ($mergedResults as $document) {
|
||||
if (null !== $maxItems && $count >= $maxItems) {
|
||||
break;
|
||||
}
|
||||
|
||||
yield $document;
|
||||
++$count;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return VectorDocument[]
|
||||
*/
|
||||
private function loadAllDocuments(): array
|
||||
{
|
||||
$statement = $this->connection->query(\sprintf(
|
||||
'SELECT id, vector, metadata FROM %s',
|
||||
$this->tableName,
|
||||
));
|
||||
|
||||
$documents = [];
|
||||
foreach ($statement->fetchAll(\PDO::FETCH_ASSOC) as $row) {
|
||||
$documents[] = new VectorDocument(
|
||||
id: $row['id'],
|
||||
vector: new Vector(json_decode($row['vector'], true)),
|
||||
metadata: new Metadata(json_decode($row['metadata'] ?? '{}', true)),
|
||||
);
|
||||
}
|
||||
|
||||
return $documents;
|
||||
}
|
||||
}
|
||||
27
src/store/src/Bridge/Sqlite/Tests/IntegrationTest.php
Normal file
27
src/store/src/Bridge/Sqlite/Tests/IntegrationTest.php
Normal file
@@ -0,0 +1,27 @@
|
||||
<?php
|
||||
|
||||
/*
|
||||
* This file is part of the Symfony package.
|
||||
*
|
||||
* (c) Fabien Potencier <fabien@symfony.com>
|
||||
*
|
||||
* For the full copyright and license information, please view the LICENSE
|
||||
* file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace Symfony\AI\Store\Bridge\Sqlite\Tests;
|
||||
|
||||
use Symfony\AI\Store\Bridge\Sqlite\Store;
|
||||
use Symfony\AI\Store\StoreInterface;
|
||||
use Symfony\AI\Store\Test\AbstractStoreIntegrationTestCase;
|
||||
|
||||
final class IntegrationTest extends AbstractStoreIntegrationTestCase
|
||||
{
|
||||
protected static function createStore(): StoreInterface
|
||||
{
|
||||
$pdo = new \PDO('sqlite::memory:');
|
||||
$pdo->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION);
|
||||
|
||||
return new Store($pdo, 'test_vectors');
|
||||
}
|
||||
}
|
||||
438
src/store/src/Bridge/Sqlite/Tests/StoreTest.php
Normal file
438
src/store/src/Bridge/Sqlite/Tests/StoreTest.php
Normal file
@@ -0,0 +1,438 @@
|
||||
<?php
|
||||
|
||||
/*
|
||||
* This file is part of the Symfony package.
|
||||
*
|
||||
* (c) Fabien Potencier <fabien@symfony.com>
|
||||
*
|
||||
* For the full copyright and license information, please view the LICENSE
|
||||
* file that was distributed with this source code.
|
||||
*/
|
||||
|
||||
namespace Symfony\AI\Store\Bridge\Sqlite\Tests;
|
||||
|
||||
use Doctrine\DBAL\Connection;
|
||||
use PHPUnit\Framework\TestCase;
|
||||
use Symfony\AI\Platform\Vector\Vector;
|
||||
use Symfony\AI\Store\Bridge\Sqlite\Store;
|
||||
use Symfony\AI\Store\Document\Metadata;
|
||||
use Symfony\AI\Store\Document\VectorDocument;
|
||||
use Symfony\AI\Store\Exception\InvalidArgumentException;
|
||||
use Symfony\AI\Store\Exception\UnsupportedQueryTypeException;
|
||||
use Symfony\AI\Store\Query\HybridQuery;
|
||||
use Symfony\AI\Store\Query\TextQuery;
|
||||
use Symfony\AI\Store\Query\VectorQuery;
|
||||
|
||||
final class StoreTest extends TestCase
|
||||
{
|
||||
public function testSetup()
|
||||
{
|
||||
$pdo = $this->createPdo();
|
||||
$store = new Store($pdo, 'test_vectors');
|
||||
$store->setup();
|
||||
|
||||
// Verify main table exists
|
||||
$result = $pdo->query("SELECT name FROM sqlite_master WHERE type='table' AND name='test_vectors'");
|
||||
$this->assertSame('test_vectors', $result->fetchColumn());
|
||||
|
||||
// Verify FTS table exists
|
||||
$result = $pdo->query("SELECT name FROM sqlite_master WHERE type='table' AND name='test_vectors_fts'");
|
||||
$this->assertSame('test_vectors_fts', $result->fetchColumn());
|
||||
}
|
||||
|
||||
public function testSetupWithUnsupportedOptions()
|
||||
{
|
||||
$this->expectException(InvalidArgumentException::class);
|
||||
|
||||
$store = $this->createStore();
|
||||
$store->setup(['unsupported' => true]);
|
||||
}
|
||||
|
||||
public function testDrop()
|
||||
{
|
||||
$pdo = $this->createPdo();
|
||||
$store = new Store($pdo, 'test_vectors');
|
||||
$store->setup();
|
||||
$store->drop();
|
||||
|
||||
$result = $pdo->query("SELECT name FROM sqlite_master WHERE type='table' AND name='test_vectors'");
|
||||
$this->assertFalse($result->fetchColumn());
|
||||
|
||||
$result = $pdo->query("SELECT name FROM sqlite_master WHERE type='table' AND name='test_vectors_fts'");
|
||||
$this->assertFalse($result->fetchColumn());
|
||||
}
|
||||
|
||||
public function testAddSingleDocument()
|
||||
{
|
||||
$pdo = $this->createPdo();
|
||||
$store = new Store($pdo, 'test_vectors');
|
||||
$store->setup();
|
||||
|
||||
$metadata = new Metadata(['name' => 'test']);
|
||||
$metadata->setText('Some text content');
|
||||
|
||||
$store->add(new VectorDocument(
|
||||
id: 'doc-1',
|
||||
vector: new Vector([0.1, 0.2, 0.3]),
|
||||
metadata: $metadata,
|
||||
));
|
||||
|
||||
$result = $pdo->query('SELECT COUNT(*) FROM test_vectors');
|
||||
$this->assertSame(1, (int) $result->fetchColumn());
|
||||
|
||||
$result = $pdo->query('SELECT COUNT(*) FROM test_vectors_fts');
|
||||
$this->assertSame(1, (int) $result->fetchColumn());
|
||||
}
|
||||
|
||||
public function testAddMultipleDocuments()
|
||||
{
|
||||
$pdo = $this->createPdo();
|
||||
$store = new Store($pdo, 'test_vectors');
|
||||
$store->setup();
|
||||
|
||||
$store->add([
|
||||
new VectorDocument('doc-1', new Vector([0.1, 0.2, 0.3]), new Metadata(['name' => 'first'])),
|
||||
new VectorDocument('doc-2', new Vector([0.4, 0.5, 0.6]), new Metadata(['name' => 'second'])),
|
||||
]);
|
||||
|
||||
$result = $pdo->query('SELECT COUNT(*) FROM test_vectors');
|
||||
$this->assertSame(2, (int) $result->fetchColumn());
|
||||
}
|
||||
|
||||
public function testAddDuplicateDocumentUpdates()
|
||||
{
|
||||
$pdo = $this->createPdo();
|
||||
$store = new Store($pdo, 'test_vectors');
|
||||
$store->setup();
|
||||
|
||||
$metadata1 = new Metadata(['name' => 'original']);
|
||||
$metadata1->setText('Original text');
|
||||
|
||||
$store->add(new VectorDocument('doc-1', new Vector([0.1, 0.2, 0.3]), $metadata1));
|
||||
|
||||
$metadata2 = new Metadata(['name' => 'updated']);
|
||||
$metadata2->setText('Updated text');
|
||||
|
||||
$store->add(new VectorDocument('doc-1', new Vector([0.4, 0.5, 0.6]), $metadata2));
|
||||
|
||||
$result = $pdo->query('SELECT COUNT(*) FROM test_vectors');
|
||||
$this->assertSame(1, (int) $result->fetchColumn());
|
||||
|
||||
$result = $pdo->query("SELECT metadata FROM test_vectors WHERE id = 'doc-1'");
|
||||
$metadata = json_decode($result->fetchColumn(), true);
|
||||
$this->assertSame('updated', $metadata['name']);
|
||||
|
||||
// FTS should also be updated (old deleted, new inserted)
|
||||
$result = $pdo->query('SELECT COUNT(*) FROM test_vectors_fts');
|
||||
$this->assertSame(1, (int) $result->fetchColumn());
|
||||
}
|
||||
|
||||
public function testRemoveWithSingleId()
|
||||
{
|
||||
$pdo = $this->createPdo();
|
||||
$store = new Store($pdo, 'test_vectors');
|
||||
$store->setup();
|
||||
|
||||
$metadata = new Metadata(['name' => 'test']);
|
||||
$metadata->setText('Some text');
|
||||
|
||||
$store->add(new VectorDocument('doc-1', new Vector([0.1, 0.2, 0.3]), $metadata));
|
||||
$store->remove('doc-1');
|
||||
|
||||
$result = $pdo->query('SELECT COUNT(*) FROM test_vectors');
|
||||
$this->assertSame(0, (int) $result->fetchColumn());
|
||||
|
||||
$result = $pdo->query('SELECT COUNT(*) FROM test_vectors_fts');
|
||||
$this->assertSame(0, (int) $result->fetchColumn());
|
||||
}
|
||||
|
||||
public function testRemoveWithMultipleIds()
|
||||
{
|
||||
$pdo = $this->createPdo();
|
||||
$store = new Store($pdo, 'test_vectors');
|
||||
$store->setup();
|
||||
|
||||
$store->add([
|
||||
new VectorDocument('doc-1', new Vector([0.1, 0.2, 0.3]), new Metadata(['name' => 'first'])),
|
||||
new VectorDocument('doc-2', new Vector([0.4, 0.5, 0.6]), new Metadata(['name' => 'second'])),
|
||||
new VectorDocument('doc-3', new Vector([0.7, 0.8, 0.9]), new Metadata(['name' => 'third'])),
|
||||
]);
|
||||
|
||||
$store->remove(['doc-1', 'doc-3']);
|
||||
|
||||
$result = $pdo->query('SELECT COUNT(*) FROM test_vectors');
|
||||
$this->assertSame(1, (int) $result->fetchColumn());
|
||||
|
||||
$result = $pdo->query('SELECT id FROM test_vectors');
|
||||
$this->assertSame('doc-2', $result->fetchColumn());
|
||||
}
|
||||
|
||||
public function testRemoveWithEmptyArray()
|
||||
{
|
||||
$pdo = $this->createPdo();
|
||||
$store = new Store($pdo, 'test_vectors');
|
||||
$store->setup();
|
||||
|
||||
$store->add(new VectorDocument('doc-1', new Vector([0.1, 0.2, 0.3]), new Metadata(['name' => 'test'])));
|
||||
$store->remove([]);
|
||||
|
||||
$result = $pdo->query('SELECT COUNT(*) FROM test_vectors');
|
||||
$this->assertSame(1, (int) $result->fetchColumn());
|
||||
}
|
||||
|
||||
public function testQueryVector()
|
||||
{
|
||||
$store = $this->createStore();
|
||||
$store->setup();
|
||||
|
||||
$store->add([
|
||||
new VectorDocument('doc-1', new Vector([1.0, 0.0, 0.0]), new Metadata(['name' => 'first'])),
|
||||
new VectorDocument('doc-2', new Vector([0.0, 1.0, 0.0]), new Metadata(['name' => 'second'])),
|
||||
new VectorDocument('doc-3', new Vector([0.0, 0.0, 1.0]), new Metadata(['name' => 'third'])),
|
||||
]);
|
||||
|
||||
$results = iterator_to_array($store->query(new VectorQuery(new Vector([1.0, 0.0, 0.0]))));
|
||||
|
||||
$this->assertCount(3, $results);
|
||||
$this->assertSame('doc-1', $results[0]->getId());
|
||||
}
|
||||
|
||||
public function testQueryVectorWithMaxItems()
|
||||
{
|
||||
$store = $this->createStore();
|
||||
$store->setup();
|
||||
|
||||
$store->add([
|
||||
new VectorDocument('doc-1', new Vector([1.0, 0.0, 0.0]), new Metadata(['name' => 'first'])),
|
||||
new VectorDocument('doc-2', new Vector([0.0, 1.0, 0.0]), new Metadata(['name' => 'second'])),
|
||||
new VectorDocument('doc-3', new Vector([0.0, 0.0, 1.0]), new Metadata(['name' => 'third'])),
|
||||
]);
|
||||
|
||||
$results = iterator_to_array($store->query(new VectorQuery(new Vector([1.0, 0.0, 0.0])), ['maxItems' => 2]));
|
||||
|
||||
$this->assertCount(2, $results);
|
||||
}
|
||||
|
||||
public function testQueryVectorWithFilter()
|
||||
{
|
||||
$store = $this->createStore();
|
||||
$store->setup();
|
||||
|
||||
$store->add([
|
||||
new VectorDocument('doc-1', new Vector([1.0, 0.0, 0.0]), new Metadata(['category' => 'a'])),
|
||||
new VectorDocument('doc-2', new Vector([0.9, 0.1, 0.0]), new Metadata(['category' => 'b'])),
|
||||
new VectorDocument('doc-3', new Vector([0.8, 0.2, 0.0]), new Metadata(['category' => 'a'])),
|
||||
]);
|
||||
|
||||
$results = iterator_to_array($store->query(
|
||||
new VectorQuery(new Vector([1.0, 0.0, 0.0])),
|
||||
['filter' => static fn (VectorDocument $doc): bool => 'a' === $doc->getMetadata()['category']],
|
||||
));
|
||||
|
||||
$this->assertCount(2, $results);
|
||||
foreach ($results as $result) {
|
||||
$this->assertSame('a', $result->getMetadata()['category']);
|
||||
}
|
||||
}
|
||||
|
||||
public function testQueryText()
|
||||
{
|
||||
$store = $this->createStore();
|
||||
$store->setup();
|
||||
|
||||
$metadata1 = new Metadata(['name' => 'first']);
|
||||
$metadata1->setText('The quick brown fox jumps over the lazy dog');
|
||||
|
||||
$metadata2 = new Metadata(['name' => 'second']);
|
||||
$metadata2->setText('Machine learning and artificial intelligence are transforming technology');
|
||||
|
||||
$metadata3 = new Metadata(['name' => 'third']);
|
||||
$metadata3->setText('The lazy cat sleeps all day');
|
||||
|
||||
$store->add([
|
||||
new VectorDocument('doc-1', new Vector([0.1, 0.2, 0.3]), $metadata1),
|
||||
new VectorDocument('doc-2', new Vector([0.4, 0.5, 0.6]), $metadata2),
|
||||
new VectorDocument('doc-3', new Vector([0.7, 0.8, 0.9]), $metadata3),
|
||||
]);
|
||||
|
||||
$results = iterator_to_array($store->query(new TextQuery('artificial intelligence')));
|
||||
|
||||
$this->assertCount(1, $results);
|
||||
$this->assertSame('doc-2', $results[0]->getId());
|
||||
}
|
||||
|
||||
public function testQueryTextMultipleTerms()
|
||||
{
|
||||
$store = $this->createStore();
|
||||
$store->setup();
|
||||
|
||||
$metadata1 = new Metadata(['name' => 'first']);
|
||||
$metadata1->setText('The quick brown fox jumps');
|
||||
|
||||
$metadata2 = new Metadata(['name' => 'second']);
|
||||
$metadata2->setText('Machine learning is great');
|
||||
|
||||
$metadata3 = new Metadata(['name' => 'third']);
|
||||
$metadata3->setText('The fox is lazy');
|
||||
|
||||
$store->add([
|
||||
new VectorDocument('doc-1', new Vector([0.1, 0.2, 0.3]), $metadata1),
|
||||
new VectorDocument('doc-2', new Vector([0.4, 0.5, 0.6]), $metadata2),
|
||||
new VectorDocument('doc-3', new Vector([0.7, 0.8, 0.9]), $metadata3),
|
||||
]);
|
||||
|
||||
$results = iterator_to_array($store->query(new TextQuery(['fox', 'learning'])));
|
||||
|
||||
$this->assertGreaterThanOrEqual(2, \count($results));
|
||||
}
|
||||
|
||||
public function testQueryTextNoResults()
|
||||
{
|
||||
$store = $this->createStore();
|
||||
$store->setup();
|
||||
|
||||
$metadata = new Metadata(['name' => 'test']);
|
||||
$metadata->setText('The quick brown fox');
|
||||
|
||||
$store->add(new VectorDocument('doc-1', new Vector([0.1, 0.2, 0.3]), $metadata));
|
||||
|
||||
$results = iterator_to_array($store->query(new TextQuery('nonexistent')));
|
||||
|
||||
$this->assertCount(0, $results);
|
||||
}
|
||||
|
||||
public function testQueryHybrid()
|
||||
{
|
||||
$store = $this->createStore();
|
||||
$store->setup();
|
||||
|
||||
$metadata1 = new Metadata(['name' => 'first']);
|
||||
$metadata1->setText('Vectors and embeddings in machine learning');
|
||||
|
||||
$metadata2 = new Metadata(['name' => 'second']);
|
||||
$metadata2->setText('Database indexing and search optimization');
|
||||
|
||||
$metadata3 = new Metadata(['name' => 'third']);
|
||||
$metadata3->setText('Artificial intelligence and neural networks');
|
||||
|
||||
$store->add([
|
||||
new VectorDocument('doc-1', new Vector([1.0, 0.0, 0.0]), $metadata1),
|
||||
new VectorDocument('doc-2', new Vector([0.0, 1.0, 0.0]), $metadata2),
|
||||
new VectorDocument('doc-3', new Vector([0.0, 0.0, 1.0]), $metadata3),
|
||||
]);
|
||||
|
||||
$results = iterator_to_array($store->query(
|
||||
new HybridQuery(new Vector([1.0, 0.0, 0.0]), 'neural networks', 0.5),
|
||||
));
|
||||
|
||||
$this->assertNotEmpty($results);
|
||||
|
||||
$foundIds = array_map(static fn (VectorDocument $doc): string|int => $doc->getId(), $results);
|
||||
|
||||
// Should find doc-1 (vector match) and/or doc-3 (text match)
|
||||
$this->assertTrue(
|
||||
\in_array('doc-1', $foundIds, true) || \in_array('doc-3', $foundIds, true),
|
||||
'HybridQuery should find either document 1 (vector match) or document 3 (text match)',
|
||||
);
|
||||
}
|
||||
|
||||
public function testSupportsAllQueryTypes()
|
||||
{
|
||||
$store = $this->createStore();
|
||||
|
||||
$this->assertTrue($store->supports(VectorQuery::class));
|
||||
$this->assertTrue($store->supports(TextQuery::class));
|
||||
$this->assertTrue($store->supports(HybridQuery::class));
|
||||
}
|
||||
|
||||
public function testUnsupportedQueryTypeThrows()
|
||||
{
|
||||
$store = $this->createStore();
|
||||
$store->setup();
|
||||
|
||||
$query = new class implements \Symfony\AI\Store\Query\QueryInterface {};
|
||||
|
||||
$this->expectException(UnsupportedQueryTypeException::class);
|
||||
|
||||
iterator_to_array($store->query($query));
|
||||
}
|
||||
|
||||
public function testFromPdo()
|
||||
{
|
||||
$pdo = $this->createPdo();
|
||||
$store = Store::fromPdo($pdo, 'test_vectors');
|
||||
|
||||
$store->setup();
|
||||
|
||||
$result = $pdo->query("SELECT name FROM sqlite_master WHERE type='table' AND name='test_vectors'");
|
||||
$this->assertSame('test_vectors', $result->fetchColumn());
|
||||
}
|
||||
|
||||
public function testFromDbalWithPdoDriver()
|
||||
{
|
||||
$pdo = $this->createPdo();
|
||||
|
||||
$connection = $this->createMock(Connection::class);
|
||||
$connection->expects($this->once())
|
||||
->method('getNativeConnection')
|
||||
->willReturn($pdo);
|
||||
|
||||
$store = Store::fromDbal($connection, 'test_vectors');
|
||||
$store->setup();
|
||||
|
||||
$result = $pdo->query("SELECT name FROM sqlite_master WHERE type='table' AND name='test_vectors'");
|
||||
$this->assertSame('test_vectors', $result->fetchColumn());
|
||||
}
|
||||
|
||||
public function testFromDbalWithNonPdoDriverThrows()
|
||||
{
|
||||
$connection = $this->createMock(Connection::class);
|
||||
$connection->expects($this->once())
|
||||
->method('getNativeConnection')
|
||||
->willReturn(new \stdClass());
|
||||
|
||||
$this->expectException(InvalidArgumentException::class);
|
||||
$this->expectExceptionMessage('Only DBAL connections using PDO driver are supported.');
|
||||
|
||||
Store::fromDbal($connection, 'test_vectors');
|
||||
}
|
||||
|
||||
public function testQueryVectorWithEmptyStore()
|
||||
{
|
||||
$store = $this->createStore();
|
||||
$store->setup();
|
||||
|
||||
$results = iterator_to_array($store->query(new VectorQuery(new Vector([1.0, 0.0, 0.0]))));
|
||||
|
||||
$this->assertCount(0, $results);
|
||||
}
|
||||
|
||||
public function testAddDocumentWithoutText()
|
||||
{
|
||||
$pdo = $this->createPdo();
|
||||
$store = new Store($pdo, 'test_vectors');
|
||||
$store->setup();
|
||||
|
||||
$store->add(new VectorDocument('doc-1', new Vector([0.1, 0.2, 0.3]), new Metadata(['name' => 'no-text'])));
|
||||
|
||||
$result = $pdo->query('SELECT COUNT(*) FROM test_vectors');
|
||||
$this->assertSame(1, (int) $result->fetchColumn());
|
||||
|
||||
// No FTS entry since no text metadata
|
||||
$result = $pdo->query('SELECT COUNT(*) FROM test_vectors_fts');
|
||||
$this->assertSame(0, (int) $result->fetchColumn());
|
||||
}
|
||||
|
||||
private function createPdo(): \PDO
|
||||
{
|
||||
$pdo = new \PDO('sqlite::memory:');
|
||||
$pdo->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION);
|
||||
|
||||
return $pdo;
|
||||
}
|
||||
|
||||
private function createStore(?\PDO $pdo = null): Store
|
||||
{
|
||||
return new Store($pdo ?? $this->createPdo(), 'test_vectors');
|
||||
}
|
||||
}
|
||||
62
src/store/src/Bridge/Sqlite/composer.json
Normal file
62
src/store/src/Bridge/Sqlite/composer.json
Normal file
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"name": "symfony/ai-sqlite-store",
|
||||
"description": "SQLite vector store bridge for Symfony AI",
|
||||
"license": "MIT",
|
||||
"type": "symfony-ai-store",
|
||||
"keywords": [
|
||||
"ai",
|
||||
"bridge",
|
||||
"sqlite",
|
||||
"store",
|
||||
"vector"
|
||||
],
|
||||
"authors": [
|
||||
{
|
||||
"name": "Christopher Hertel",
|
||||
"email": "mail@christopher-hertel.de"
|
||||
},
|
||||
{
|
||||
"name": "Oskar Stark",
|
||||
"email": "oskarstark@googlemail.com"
|
||||
},
|
||||
{
|
||||
"name": "Symfony Community",
|
||||
"homepage": "https://symfony.com/contributors"
|
||||
}
|
||||
],
|
||||
"require": {
|
||||
"php": ">=8.2",
|
||||
"ext-pdo": "*",
|
||||
"ext-pdo_sqlite": "*",
|
||||
"symfony/ai-platform": "^0.5|^0.6",
|
||||
"symfony/ai-store": "^0.5|^0.6"
|
||||
},
|
||||
"require-dev": {
|
||||
"doctrine/dbal": "^3.3|^4.0",
|
||||
"phpstan/phpstan": "^2.1",
|
||||
"phpstan/phpstan-phpunit": "^2.0",
|
||||
"phpstan/phpstan-strict-rules": "^2.0",
|
||||
"phpunit/phpunit": "^11.5.53"
|
||||
},
|
||||
"minimum-stability": "dev",
|
||||
"autoload": {
|
||||
"psr-4": {
|
||||
"Symfony\\AI\\Store\\Bridge\\Sqlite\\": ""
|
||||
}
|
||||
},
|
||||
"autoload-dev": {
|
||||
"psr-4": {
|
||||
"Symfony\\AI\\PHPStan\\": "../../../../../.phpstan/",
|
||||
"Symfony\\AI\\Store\\Bridge\\Sqlite\\Tests\\": "Tests/"
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"sort-packages": true
|
||||
},
|
||||
"extra": {
|
||||
"thanks": {
|
||||
"name": "symfony/ai",
|
||||
"url": "https://github.com/symfony/ai"
|
||||
}
|
||||
}
|
||||
}
|
||||
28
src/store/src/Bridge/Sqlite/phpstan.dist.neon
Normal file
28
src/store/src/Bridge/Sqlite/phpstan.dist.neon
Normal file
@@ -0,0 +1,28 @@
|
||||
includes:
|
||||
- vendor/phpstan/phpstan-phpunit/extension.neon
|
||||
- ../../../../../.phpstan/extension.neon
|
||||
|
||||
parameters:
|
||||
level: 6
|
||||
paths:
|
||||
- .
|
||||
- Tests/
|
||||
excludePaths:
|
||||
- vendor/
|
||||
treatPhpDocTypesAsCertain: false
|
||||
ignoreErrors:
|
||||
-
|
||||
message: "#^Method .*::test.*\\(\\) has no return type specified\\.$#"
|
||||
-
|
||||
message: '#^Call to( static)? method PHPUnit\\Framework\\Assert::.* will always evaluate to true\.$#'
|
||||
reportUnmatched: false
|
||||
-
|
||||
identifier: 'symfonyAi.forbidNativeException'
|
||||
path: Tests/*
|
||||
reportUnmatched: false
|
||||
|
||||
services:
|
||||
- # Conditionally enabled by bleeding edge in phpstan/phpstan-phpunit 2.x
|
||||
class: PHPStan\Type\PHPUnit\DataProviderReturnTypeIgnoreExtension
|
||||
tags:
|
||||
- phpstan.ignoreErrorExtension
|
||||
33
src/store/src/Bridge/Sqlite/phpunit.xml.dist
Normal file
33
src/store/src/Bridge/Sqlite/phpunit.xml.dist
Normal file
@@ -0,0 +1,33 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:noNamespaceSchemaLocation="vendor/phpunit/phpunit/phpunit.xsd"
|
||||
backupGlobals="false"
|
||||
colors="true"
|
||||
bootstrap="vendor/autoload.php"
|
||||
failOnDeprecation="true"
|
||||
failOnRisky="true"
|
||||
failOnWarning="true"
|
||||
executionOrder="random"
|
||||
>
|
||||
<php>
|
||||
<ini name="error_reporting" value="-1" />
|
||||
</php>
|
||||
|
||||
<testsuites>
|
||||
<testsuite name="Symfony AI SQLite Store Test Suite">
|
||||
<directory>./Tests/</directory>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
|
||||
<source ignoreSuppressionOfDeprecations="true">
|
||||
<include>
|
||||
<directory>./</directory>
|
||||
</include>
|
||||
<exclude>
|
||||
<directory>./Resources</directory>
|
||||
<directory>./Tests</directory>
|
||||
<directory>./vendor</directory>
|
||||
</exclude>
|
||||
</source>
|
||||
</phpunit>
|
||||
Reference in New Issue
Block a user