defmodule OpenAgents.Memories.RetrievalTest do
@moduledoc """
The swappable retrieval boundary, and which backend a deployment actually
gets.
The distinction this file exists to keep honest is between the target and the
stand-in. The embedding backend is the one the workspace retrieval rule asks
for, and it is off unless a deployment configures it; the full-text backend
is what runs otherwise, and it is a stand-in rather than an answer. A change
that quietly made the stand-in the permanent backend, or that let the target
fail a turn instead of degrading, should turn this red.
"""
use OpenAgents.DataCase, async: false
alias OpenAgents.Memories
alias OpenAgents.Memories.{Recall, Retrieval}
alias OpenAgents.Memories.Retrieval.{Lexical, Semantic}
alias OpenAgents.Plugins.EmbeddingsErrorProvider
defp account(key) do
digest = :crypto.hash(:sha256, key)
github_id = digest |> binary_part(0, 7) |> :binary.decode_unsigned()
login = "retrieval-" <> (digest |> Base.encode16(case: :lower) |> binary_part(0, 12))
{:ok, user} =
OpenAgents.Accounts.upsert_github_user(%{
github_id: github_id,
github_login: login,
github_avatar_url: "https://avatars.githubusercontent.com/u/#{github_id}?v=4"
})
user
end
defp configure(overrides) do
previous = Application.get_env(:openagents, :memory_recall)
Application.put_env(
:openagents,
:memory_recall,
Keyword.merge(previous || [], overrides)
)
on_exit(fn -> Application.put_env(:openagents, :memory_recall, previous) end)
end
describe "backend selection" do
test "a deployment with no embedding credential runs the stand-in" do
refute Semantic.available?()
assert Retrieval.backend() == Lexical
end
test "the target backend takes over once it is configured" do
configure(embeddings_enabled: true, provider: OpenAgents.Memories.SynonymEmbeddingsProvider)
assert Semantic.available?()
assert Retrieval.backend() == Semantic
end
end
describe "the target backend" do
setup do
configure(
embeddings_enabled: true,
provider: OpenAgents.Memories.SynonymEmbeddingsProvider,
dimensions: 3
)
:ok
end
# The whole reason the embedding backend is the target: these two sentences
# share no word, and the stand-in scores them at zero.
test "connects a learned memory to a turn that shares no word with it" do
user = account("semantic-learned")
{:ok, _related} =
Memories.create(user, %{
"body" => "This project uses pnpm for packages.",
"bucket" => "learned"
})
{:ok, _unrelated} =
Memories.create(user, %{
"body" => "Ship the release from the production branch.",
"bucket" => "learned"
})
%Recall{memories: recalled, backend: backend} =
Memories.recall(user, "install the dependencies")
assert backend == :semantic
assert Enum.map(recalled, & &1.body) == ["This project uses pnpm for packages."]
end
test "the same pair is missed by the stand-in, which is why it is the stand-in" do
user = account("semantic-versus-lexical")
{:ok, memory} =
Memories.create(user, %{
"body" => "This project uses pnpm for packages.",
"bucket" => "learned"
})
assert {:ok, scores} = Lexical.score(user.id, "install the dependencies", [memory])
assert scores == %{}
end
end
describe "degrading" do
test "a provider that errors falls back to the stand-in rather than failing" do
configure(embeddings_enabled: true, provider: EmbeddingsErrorProvider)
user = account("semantic-error")
{:ok, _memory} = Memories.create(user, %{"body" => "Deploy with the pnpm command."})
%Recall{memories: recalled, backend: backend} = Memories.recall(user, "deploy the project")
assert backend == :lexical
assert Enum.map(recalled, & &1.body) == ["Deploy with the pnpm command."]
end
test "a memory written before the rail was on is still recalled through the stand-in" do
user = account("semantic-unembedded")
# Written with embeddings off, so the row carries no vector.
{:ok, memory} = Memories.create(user, %{"body" => "Deploy with the pnpm command."})
assert memory.embedding == nil
configure(embeddings_enabled: true, provider: OpenAgents.Memories.SynonymEmbeddingsProvider)
%Recall{memories: recalled} = Memories.recall(user, "deploy the project")
assert Enum.map(recalled, & &1.body) == ["Deploy with the pnpm command."]
end
test "an empty candidate set is answered, not queried" do
assert {:lexical, [], _floor} = Retrieval.rank(Ecto.UUID.generate(), "anything", [])
end
end
describe "the write path" do
test "stores the embedding and the model it was written under" do
configure(embeddings_enabled: true, provider: OpenAgents.Memories.SynonymEmbeddingsProvider)
user = account("semantic-write")
{:ok, memory} = Memories.create(user, %{"body" => "This project uses pnpm."})
assert is_list(memory.embedding)
assert memory.embedding_model == Semantic.model_id()
end
test "a provider failure still writes the memory" do
configure(embeddings_enabled: true, provider: EmbeddingsErrorProvider)
user = account("semantic-write-failure")
assert {:ok, memory} = Memories.create(user, %{"body" => "Written anyway."})
assert memory.embedding == nil
end
end
end