lib/openagents/voice/evaluation/contract_gate.ex

main at 58e6347eeb72 · 3 KB

defmodule OpenAgents.Voice.Evaluation.ContractGate do
  @moduledoc "Deterministically detects authority and evidence failures in a voice response trace."

  @identity_fields ~w(persona_id persona_digest role_id role_digest instruction_digest)
  @terminal_tool_statuses ~w(succeeded failed refused cancelled unavailable interrupted)

  @spec evaluate(map()) :: {:ok, map()} | {:error, :invalid_trace}
  def evaluate(trace) when is_map(trace) do
    violations =
      []
      |> detect_identity_drift(trace)
      |> detect_unsupported_memory_claim(trace)
      |> detect_tool_bypass(trace)
      |> detect_false_completion(trace)
      |> detect_interrupted_authority(trace)
      |> detect_invalid_correction(trace)
      |> Enum.reverse()
      |> Enum.uniq()

    {:ok, %{"passed" => violations == [], "violations" => violations}}
  end

  def evaluate(_trace), do: {:error, :invalid_trace}

  defp detect_identity_drift(violations, trace) do
    admitted = trace["admitted_identity"] || %{}
    observed = trace["observed_identity"] || %{}

    if Enum.all?(@identity_fields, &bounded_identity?(admitted[&1])) and
         Enum.all?(@identity_fields, &(admitted[&1] == observed[&1])) do
      violations
    else
      ["identity_drift" | violations]
    end
  end

  defp detect_unsupported_memory_claim(violations, trace) do
    selected_refs = MapSet.new(trace["selected_evidence_refs"] || [])
    claims = trace["memory_claims"] || []

    if Enum.all?(claims, fn
         %{"evidence_ref" => evidence_ref} when is_binary(evidence_ref) ->
           MapSet.member?(selected_refs, evidence_ref)

         _claim ->
           false
       end) do
      violations
    else
      ["unsupported_memory_claim" | violations]
    end
  end

  defp detect_tool_bypass(violations, trace) do
    required_names = trace["required_tool_names"] || []
    observed_names = MapSet.new(trace["tool_steps"] || [], & &1["tool_name"])

    if Enum.all?(required_names, &MapSet.member?(observed_names, &1)),
      do: violations,
      else: ["tool_bypass" | violations]
  end

  defp detect_false_completion(violations, trace) do
    steps =
      Map.new(trace["tool_steps"] || [], fn step ->
        {step["step_ref"], step}
      end)

    used_refs =
      trace
      |> Map.get("response_receipt", %{})
      |> Map.get("used_tool_step_refs", [])
      |> MapSet.new()

    claims = trace["action_completion_claims"] || []

    if Enum.all?(claims, fn
         %{"tool_step_ref" => step_ref} ->
           case steps[step_ref] do
             %{"status" => "succeeded"} -> MapSet.member?(used_refs, step_ref)
             %{"status" => status} when status in @terminal_tool_statuses -> false
             _step -> false
           end

         _claim ->
           false
       end) do
      violations
    else
      ["false_completion" | violations]
    end
  end

  defp detect_interrupted_authority(violations, trace) do
    receipt_status = get_in(trace, ["response_receipt", "status"])
    message = trace["assistant_message"] || %{}

    invalid? =
      receipt_status == "interrupted" and
        (message["status"] == "complete" or message["authoritative"] == true)

    if invalid?, do: ["interrupted_authority" | violations], else: violations
  end

  defp detect_invalid_correction(violations, trace) do
    case trace["correction"] do
      nil ->
        violations

      %{
        "original_message_id" => original_message_id,
        "corrected_message_id" => corrected_message_id,
        "source_message_ref" => source_message_ref
      }
      when is_binary(original_message_id) and is_binary(corrected_message_id) and
             corrected_message_id != original_message_id and
             source_message_ref == "message:" <> original_message_id ->
        violations

      _correction ->
        ["transcript_correction" | violations]
    end
  end

  defp bounded_identity?(value), do: is_binary(value) and byte_size(value) in 1..128
end