| 534 |
534
|
|
end
|
| 535 |
535
|
|
end
|
| 536 |
536
|
|
|
|
537
|
+ |
describe "POST /api/v1/threads/:thread_id/report" do
|
|
538
|
+ |
setup %{conn: conn} do
|
|
539
|
+ |
authenticated = put_chat_api_token(conn, "thread-report")
|
|
540
|
+ |
|
|
541
|
+ |
created =
|
|
542
|
+ |
authenticated
|
|
543
|
+ |
|> post(~p"/api/v1/threads", %{"objective" => "Answer, then say so."})
|
|
544
|
+ |
|> json_response(201)
|
|
545
|
+ |
|
|
546
|
+ |
%{
|
|
547
|
+ |
authenticated: authenticated,
|
|
548
|
+ |
id: created["thread"]["id"],
|
|
549
|
+ |
token: created["grant"]["token"]
|
|
550
|
+ |
}
|
|
551
|
+ |
end
|
|
552
|
+ |
|
|
553
|
+ |
test "a thread that reported is recorded as having reported, not cancelled", %{
|
|
554
|
+ |
authenticated: conn,
|
|
555
|
+ |
id: id,
|
|
556
|
+ |
token: token
|
|
557
|
+ |
} do
|
|
558
|
+ |
body =
|
|
559
|
+ |
conn
|
|
560
|
+ |
|> post(~p"/api/v1/threads/#{id}/report", %{
|
|
561
|
+ |
"status" => "succeeded",
|
|
562
|
+ |
"report" => "The answer is 4."
|
|
563
|
+ |
})
|
|
564
|
+ |
|> json_response(200)
|
|
565
|
+ |
|
|
566
|
+ |
assert body["thread"]["status"] == "succeeded"
|
|
567
|
+ |
assert body["thread"]["error_code"] == nil
|
|
568
|
+ |
assert body["thread"]["report"] == "The answer is 4."
|
|
569
|
+ |
assert body["thread"]["report_type"] == "outcome"
|
|
570
|
+ |
|
|
571
|
+ |
# Reporting revokes, exactly as cancelling does: authority does not
|
|
572
|
+ |
# outlive the thread's end (THREAD-001).
|
|
573
|
+ |
assert {:error, :grant_revoked} = Inference.resolve(token)
|
|
574
|
+ |
assert body["grant"]["status"] == "revoked"
|
|
575
|
+ |
end
|
|
576
|
+ |
|
|
577
|
+ |
test "the server never guesses the outcome: a report with no status is refused", %{
|
|
578
|
+ |
authenticated: conn,
|
|
579
|
+ |
id: id
|
|
580
|
+ |
} do
|
|
581
|
+ |
body =
|
|
582
|
+ |
conn
|
|
583
|
+ |
|> post(~p"/api/v1/threads/#{id}/report", %{"report" => "Something happened."})
|
|
584
|
+ |
|> json_response(422)
|
|
585
|
+ |
|
|
586
|
+ |
assert body["code"] == "validation_failed"
|
|
587
|
+ |
assert body["errors"]["status"] != nil
|
|
588
|
+ |
|
|
589
|
+ |
assert conn
|
|
590
|
+ |
|> get(~p"/api/v1/threads/#{id}")
|
|
591
|
+ |
|> json_response(200)
|
|
592
|
+ |
|> get_in([
|
|
593
|
+ |
"thread",
|
|
594
|
+ |
"status"
|
|
595
|
+ |
]) == "open"
|
|
596
|
+ |
end
|
|
597
|
+ |
|
|
598
|
+ |
test "a run that failed cannot be recorded as a success", %{authenticated: conn, id: id} do
|
|
599
|
+ |
body =
|
|
600
|
+ |
conn
|
|
601
|
+ |
|> post(~p"/api/v1/threads/#{id}/report", %{
|
|
602
|
+ |
"status" => "succeeded",
|
|
603
|
+ |
"report" => "It worked.",
|
|
604
|
+ |
"error_code" => "max_steps"
|
|
605
|
+ |
})
|
|
606
|
+ |
|> json_response(422)
|
|
607
|
+ |
|
|
608
|
+ |
assert body["code"] == "validation_failed"
|
|
609
|
+ |
assert body["errors"]["error_code"] != nil
|
|
610
|
+ |
end
|
|
611
|
+ |
|
|
612
|
+ |
test "a failure has to name why", %{authenticated: conn, id: id} do
|
|
613
|
+ |
body =
|
|
614
|
+ |
conn
|
|
615
|
+ |
|> post(~p"/api/v1/threads/#{id}/report", %{
|
|
616
|
+ |
"status" => "failed",
|
|
617
|
+ |
"report" => "It did not work."
|
|
618
|
+ |
})
|
|
619
|
+ |
|> json_response(422)
|
|
620
|
+ |
|
|
621
|
+ |
assert body["code"] == "validation_failed"
|
|
622
|
+ |
assert body["errors"]["error_code"] != nil
|
|
623
|
+ |
end
|
|
624
|
+ |
|
|
625
|
+ |
test "a failed run is recorded as failed, with its reason", %{authenticated: conn, id: id} do
|
|
626
|
+ |
body =
|
|
627
|
+ |
conn
|
|
628
|
+ |
|> post(~p"/api/v1/threads/#{id}/report", %{
|
|
629
|
+ |
"status" => "failed",
|
|
630
|
+ |
"report" => "The turn budget ran out before an answer.",
|
|
631
|
+ |
"error_code" => "max_steps"
|
|
632
|
+ |
})
|
|
633
|
+ |
|> json_response(200)
|
|
634
|
+ |
|
|
635
|
+ |
assert body["thread"]["status"] == "failed"
|
|
636
|
+ |
assert body["thread"]["error_code"] == "max_steps"
|
|
637
|
+ |
assert body["thread"]["report_type"] == "failure"
|
|
638
|
+ |
end
|
|
639
|
+ |
|
|
640
|
+ |
test "an interrupted run reports as cancelled, naming the interruption", %{
|
|
641
|
+ |
authenticated: conn,
|
|
642
|
+ |
id: id
|
|
643
|
+ |
} do
|
|
644
|
+ |
body =
|
|
645
|
+ |
conn
|
|
646
|
+ |
|> post(~p"/api/v1/threads/#{id}/report", %{
|
|
647
|
+ |
"status" => "cancelled",
|
|
648
|
+ |
"report" => "The operator interrupted the session.",
|
|
649
|
+ |
"error_code" => "interrupted"
|
|
650
|
+ |
})
|
|
651
|
+ |
|> json_response(200)
|
|
652
|
+ |
|
|
653
|
+ |
assert body["thread"]["status"] == "cancelled"
|
|
654
|
+ |
assert body["thread"]["error_code"] == "interrupted"
|
|
655
|
+ |
end
|
|
656
|
+ |
|
|
657
|
+ |
test "a status outside the terminal three is refused", %{authenticated: conn, id: id} do
|
|
658
|
+ |
body =
|
|
659
|
+ |
conn
|
|
660
|
+ |
|> post(~p"/api/v1/threads/#{id}/report", %{
|
|
661
|
+ |
"status" => "open",
|
|
662
|
+ |
"report" => "Still going."
|
|
663
|
+ |
})
|
|
664
|
+ |
|> json_response(422)
|
|
665
|
+ |
|
|
666
|
+ |
assert body["errors"]["status"] != nil
|
|
667
|
+ |
end
|
|
668
|
+ |
|
|
669
|
+ |
test "a blank report is refused", %{authenticated: conn, id: id} do
|
|
670
|
+ |
body =
|
|
671
|
+ |
conn
|
|
672
|
+ |
|> post(~p"/api/v1/threads/#{id}/report", %{"status" => "succeeded", "report" => " "})
|
|
673
|
+ |
|> json_response(422)
|
|
674
|
+ |
|
|
675
|
+ |
assert body["errors"]["report"] != nil
|
|
676
|
+ |
end
|
|
677
|
+ |
|
|
678
|
+ |
test "an at-least-once client may resend the same report", %{authenticated: conn, id: id} do
|
|
679
|
+ |
report = %{"status" => "succeeded", "report" => "The answer is 4."}
|
|
680
|
+ |
|
|
681
|
+ |
assert conn |> post(~p"/api/v1/threads/#{id}/report", report) |> json_response(200)
|
|
682
|
+ |
body = conn |> post(~p"/api/v1/threads/#{id}/report", report) |> json_response(200)
|
|
683
|
+ |
|
|
684
|
+ |
assert body["thread"]["status"] == "succeeded"
|
|
685
|
+ |
end
|
|
686
|
+ |
|
|
687
|
+ |
test "a second, different report is refused rather than overwriting the first", %{
|
|
688
|
+ |
authenticated: conn,
|
|
689
|
+ |
id: id
|
|
690
|
+ |
} do
|
|
691
|
+ |
assert conn
|
|
692
|
+ |
|> post(~p"/api/v1/threads/#{id}/report", %{
|
|
693
|
+ |
"status" => "succeeded",
|
|
694
|
+ |
"report" => "The answer is 4."
|
|
695
|
+ |
})
|
|
696
|
+ |
|> json_response(200)
|
|
697
|
+ |
|
|
698
|
+ |
body =
|
|
699
|
+ |
conn
|
|
700
|
+ |
|> post(~p"/api/v1/threads/#{id}/report", %{
|
|
701
|
+ |
"status" => "failed",
|
|
702
|
+ |
"report" => "Actually it broke.",
|
|
703
|
+ |
"error_code" => "max_steps"
|
|
704
|
+ |
})
|
|
705
|
+ |
|> json_response(422)
|
|
706
|
+ |
|
|
707
|
+ |
assert body["code"] == "thread_terminal"
|
|
708
|
+ |
|
|
709
|
+ |
standing = conn |> get(~p"/api/v1/threads/#{id}") |> json_response(200)
|
|
710
|
+ |
assert standing["thread"]["status"] == "succeeded"
|
|
711
|
+ |
end
|
|
712
|
+ |
|
|
713
|
+ |
test "another account cannot report on a thread it did not open", %{conn: conn, id: id} do
|
|
714
|
+ |
body =
|
|
715
|
+ |
conn
|
|
716
|
+ |
|> put_chat_api_token("thread-report-stranger")
|
|
717
|
+ |
|> post(~p"/api/v1/threads/#{id}/report", %{
|
|
718
|
+ |
"status" => "succeeded",
|
|
719
|
+ |
"report" => "Not mine."
|
|
720
|
+ |
})
|
|
721
|
+ |
|> json_response(404)
|
|
722
|
+ |
|
|
723
|
+ |
assert body["code"] == "not_found"
|
|
724
|
+ |
end
|
|
725
|
+ |
end
|
|
726
|
+ |
|
| 537 |
727
|
|
describe "spending a thread's grant" do
|
| 538 |
728
|
|
test "the grant reaches the model exactly as a conversation-fenced one does", %{conn: conn} do
|
| 539 |
729
|
|
authenticated = put_chat_api_token(conn, "thread-spend")
|