|
1
|
+ |
defmodule OpenAgents.ContinualLearning do
|
|
2
|
+ |
@moduledoc """
|
|
3
|
+ |
Bounded continual-learning jobs over verified licensed datasets.
|
|
4
|
+ |
|
|
5
|
+ |
One named internal buyer starts a job that names a versioned objective, an
|
|
6
|
+ |
admitted base model, exact licensed dataset references, an evaluation corpus,
|
|
7
|
+ |
a budget, a runtime class, and a stopping policy. Admission resolves every
|
|
8
|
+ |
dataset through `OpenAgents.ArtifactCatalog`, so a job holds the exact
|
|
9
|
+ |
artifact, provenance, license, and listing digests it trained on, and a
|
|
10
|
+ |
removed listing, an expired license, a license that does not admit training,
|
|
11
|
+ |
or a buyer class the listing was not licensed to refuses before any capacity
|
|
12
|
+ |
is spent. Fleet admission is `OpenAgents.Capacity.match/2`: this lane adds no
|
|
13
|
+ |
second scheduler, and the run itself is an ordinary `work_jobs` row of kind
|
|
14
|
+ |
`continual_learning` driven by `OpenAgents.Work.ContinualLearningServer`.
|
|
15
|
+ |
|
|
16
|
+ |
Every round writes a durable checkpoint before it is counted, so resume and
|
|
17
|
+ |
replay are different acts: a resume continues the surviving checkpoint chain
|
|
18
|
+ |
under the same admission digest, and a replay is a new job that starts from
|
|
19
|
+ |
round zero. A lost checkpoint refuses the resume instead of retraining
|
|
20
|
+ |
silently.
|
|
21
|
+ |
|
|
22
|
+ |
Evaluation is graded through `OpenAgents.AcceptedOutcome`, under the admitted
|
|
23
|
+ |
evaluator policy, so a failed, unevidenced, or non-independent evaluation
|
|
24
|
+ |
cannot produce a qualified artifact. A qualified artifact binds the exact base
|
|
25
|
+ |
model, dataset, code, configuration, checkpoint, and evaluation digests, and
|
|
26
|
+ |
the job's settlement-ready receipt names the buyer, the unit, the amount, and
|
|
27
|
+ |
the treasury policy without moving money.
|
|
28
|
+ |
|
|
29
|
+ |
See `INVARIANTS.md`, CONTINUAL-001.
|
|
30
|
+ |
"""
|
|
31
|
+ |
|
|
32
|
+ |
import Ecto.Query
|
|
33
|
+ |
|
|
34
|
+ |
alias OpenAgents.AcceptedOutcome
|
|
35
|
+ |
alias OpenAgents.Accounts
|
|
36
|
+ |
alias OpenAgents.Accounts.User
|
|
37
|
+ |
alias OpenAgents.ArtifactCatalog
|
|
38
|
+ |
alias OpenAgents.Capacity
|
|
39
|
+ |
alias OpenAgents.ContinualLearning.Artifact
|
|
40
|
+ |
alias OpenAgents.ContinualLearning.Bounds
|
|
41
|
+ |
alias OpenAgents.ContinualLearning.Checkpoint
|
|
42
|
+ |
alias OpenAgents.ContinualLearning.Job
|
|
43
|
+ |
alias OpenAgents.ContinualLearning.Receipt
|
|
44
|
+ |
alias OpenAgents.Provenance.Canonical
|
|
45
|
+ |
alias OpenAgents.Repo
|
|
46
|
+ |
alias OpenAgents.Settlement
|
|
47
|
+ |
alias OpenAgents.Work
|
|
48
|
+ |
|
|
49
|
+ |
@active_statuses ~w(queued running)
|
|
50
|
+ |
@evaluation_purpose "evaluation"
|
|
51
|
+ |
@training_purpose "delivery"
|
|
52
|
+ |
|
|
53
|
+ |
# ── admission ──────────────────────────────────────────────────────────────
|
|
54
|
+ |
|
|
55
|
+ |
@doc """
|
|
56
|
+ |
Admits and starts one continual-learning job for the named buyer.
|
|
57
|
+ |
|
|
58
|
+ |
Returns `{:ok, job}` with a queued job whose run is already supervised, or a
|
|
59
|
+ |
typed refusal.
|
|
60
|
+ |
"""
|
|
61
|
+ |
@spec start(User.t(), map()) :: {:ok, Job.t()} | {:error, term()}
|
|
62
|
+ |
def start(%User{} = user, attributes) when is_map(attributes) do
|
|
63
|
+ |
with :ok <- feature_enabled(),
|
|
64
|
+ |
:ok <- operator(user),
|
|
65
|
+ |
{:ok, buyer_ref} <- buyer_ref(attributes),
|
|
66
|
+ |
{:ok, buyer_class} <- buyer_class(),
|
|
67
|
+ |
{:ok, objective} <- objective(attributes),
|
|
68
|
+ |
{:ok, objective_version} <- objective_version(attributes),
|
|
69
|
+ |
{:ok, base_model} <- base_model(attributes),
|
|
70
|
+ |
{:ok, training_code_digest} <- training_code_digest(),
|
|
71
|
+ |
{:ok, configuration} <- configuration(attributes),
|
|
72
|
+ |
{:ok, runtime_class} <- runtime_class(attributes),
|
|
73
|
+ |
{:ok, conversation_id} <- identifier(attributes, :conversation_id),
|
|
74
|
+ |
{:ok, owner_visitor_id} <- identifier(attributes, :owner_visitor_id),
|
|
75
|
+ |
{:ok, datasets} <- datasets(attributes, buyer_ref, buyer_class, runtime_class),
|
|
76
|
+ |
{:ok, evaluation} <- evaluation(attributes, buyer_ref, buyer_class, runtime_class),
|
|
77
|
+ |
{:ok, budget} <- budget(attributes),
|
|
78
|
+ |
{:ok, stopping_policy} <- stopping_policy(attributes),
|
|
79
|
+ |
:ok <- concurrency(),
|
|
80
|
+ |
{:ok, capacity_receipt} <- capacity(user, runtime_class, budget, stopping_policy) do
|
|
81
|
+ |
admission = %{
|
|
82
|
+ |
buyer_ref: buyer_ref,
|
|
83
|
+ |
buyer_class: buyer_class,
|
|
84
|
+ |
objective: objective,
|
|
85
|
+ |
objective_version: objective_version,
|
|
86
|
+ |
base_model_ref: base_model.ref,
|
|
87
|
+ |
base_model_digest: base_model.digest,
|
|
88
|
+ |
training_code_digest: training_code_digest,
|
|
89
|
+ |
configuration: configuration,
|
|
90
|
+ |
configuration_digest: Canonical.digest!(configuration),
|
|
91
|
+ |
datasets: datasets,
|
|
92
|
+ |
evaluation: evaluation,
|
|
93
|
+ |
budget: budget,
|
|
94
|
+ |
runtime_class: runtime_class,
|
|
95
|
+ |
capacity_receipt: capacity_receipt,
|
|
96
|
+ |
stopping_policy: stopping_policy,
|
|
97
|
+ |
replay_of_id: Map.get(attributes, :replay_of_id)
|
|
98
|
+ |
}
|
|
99
|
+ |
|
|
100
|
+ |
with {:ok, job} <- insert_job(admission),
|
|
101
|
+ |
{:ok, started} <-
|
|
102
|
+ |
launch(job, conversation_id, owner_visitor_id, "admission") do
|
|
103
|
+ |
{:ok, started}
|
|
104
|
+ |
end
|
|
105
|
+ |
end
|
|
106
|
+ |
end
|
|
107
|
+ |
|
|
108
|
+ |
def start(_user, _attributes), do: {:error, :operator_required}
|
|
109
|
+ |
|
|
110
|
+ |
@doc "One job the buyer may read, or a typed refusal."
|
|
111
|
+ |
@spec get(User.t(), String.t()) :: {:ok, Job.t()} | {:error, term()}
|
|
112
|
+ |
def get(%User{} = user, id) when is_binary(id) do
|
|
113
|
+ |
with :ok <- operator(user), do: fetch(id)
|
|
114
|
+ |
end
|
|
115
|
+ |
|
|
116
|
+ |
def get(_user, _id), do: {:error, :operator_required}
|
|
117
|
+ |
|
|
118
|
+ |
@doc "The buyer's most recent jobs, newest first, bounded."
|
|
119
|
+ |
@spec list(User.t(), pos_integer()) :: {:ok, [Job.t()]} | {:error, term()}
|
|
120
|
+ |
def list(%User{} = user, limit \\ 50) do
|
|
121
|
+ |
with :ok <- operator(user) do
|
|
122
|
+ |
bounded = min(max(limit, 1), 200)
|
|
123
|
+ |
|
|
124
|
+ |
buyer_ref = Bounds.buyer_ref()
|
|
125
|
+ |
|
|
126
|
+ |
{:ok,
|
|
127
|
+ |
Job
|
|
128
|
+ |
|> where([job], job.buyer_ref == ^buyer_ref)
|
|
129
|
+ |
|> order_by([job], desc: job.inserted_at)
|
|
130
|
+ |
|> limit(^bounded)
|
|
131
|
+ |
|> Repo.all()}
|
|
132
|
+ |
end
|
|
133
|
+ |
end
|
|
134
|
+ |
|
|
135
|
+ |
# ── lifecycle ──────────────────────────────────────────────────────────────
|
|
136
|
+ |
|
|
137
|
+ |
@doc """
|
|
138
|
+ |
Cancels one active job.
|
|
139
|
+ |
|
|
140
|
+ |
The durable row reaches `cancelled` here, so the round loop stops at its next
|
|
141
|
+ |
boundary even when the worker is already gone.
|
|
142
|
+ |
"""
|
|
143
|
+ |
@spec cancel(User.t(), String.t()) :: {:ok, Job.t()} | {:error, term()}
|
|
144
|
+ |
def cancel(%User{} = user, id) when is_binary(id) do
|
|
145
|
+ |
with :ok <- operator(user),
|
|
146
|
+ |
{:ok, job} <- fetch(id) do
|
|
147
|
+ |
cond do
|
|
148
|
+ |
job.status == "cancelled" ->
|
|
149
|
+ |
{:ok, job}
|
|
150
|
+ |
|
|
151
|
+ |
Job.terminal?(job) ->
|
|
152
|
+ |
{:error, :not_cancellable}
|
|
153
|
+ |
|
|
154
|
+ |
true ->
|
|
155
|
+ |
if job.work_job_id, do: Work.cancel_job(job.work_job_id)
|
|
156
|
+ |
terminalize(job, "cancelled", "cancelled")
|
|
157
|
+ |
end
|
|
158
|
+ |
end
|
|
159
|
+ |
end
|
|
160
|
+ |
|
|
161
|
+ |
def cancel(_user, _id), do: {:error, :operator_required}
|
|
162
|
+ |
|
|
163
|
+ |
@doc """
|
|
164
|
+ |
Resumes one interrupted or budget-exhausted job from its surviving checkpoint.
|
|
165
|
+ |
|
|
166
|
+ |
A resume is not a replay: the job keeps its admission digest, its receipt
|
|
167
|
+ |
chain, and its checkpoints, and continues at the next round. It refuses when
|
|
168
|
+ |
the checkpoint is gone, when a dataset's license no longer admits the job, or
|
|
169
|
+ |
when the fleet cannot admit the runtime class again.
|
|
170
|
+ |
"""
|
|
171
|
+ |
@spec resume(User.t(), String.t(), map()) :: {:ok, Job.t()} | {:error, term()}
|
|
172
|
+ |
def resume(%User{} = user, id, attributes \\ %{}) do
|
|
173
|
+ |
with :ok <- feature_enabled(),
|
|
174
|
+ |
:ok <- operator(user),
|
|
175
|
+ |
{:ok, job} <- fetch(id),
|
|
176
|
+ |
:ok <- resumable(job),
|
|
177
|
+ |
{:ok, checkpoint} <- surviving_checkpoint(job),
|
|
178
|
+ |
:ok <- rounds_remaining(job),
|
|
179
|
+ |
:ok <- budget_remaining(job),
|
|
180
|
+ |
:ok <- reverify_datasets(job),
|
|
181
|
+ |
{:ok, capacity_receipt} <-
|
|
182
|
+ |
capacity(user, job.runtime_class, job.budget, job.stopping_policy),
|
|
183
|
+ |
{:ok, conversation_id, owner_visitor_id} <- previous_surface(job, attributes),
|
|
184
|
+ |
{:ok, resumed} <- mark_resumed(job, checkpoint, capacity_receipt) do
|
|
185
|
+ |
launch(resumed, conversation_id, owner_visitor_id, "resume")
|
|
186
|
+ |
end
|
|
187
|
+ |
end
|
|
188
|
+ |
|
|
189
|
+ |
@doc """
|
|
190
|
+ |
Replays one job as a new job under the same admitted inputs.
|
|
191
|
+ |
|
|
192
|
+ |
A replay re-resolves every licensed dataset and the fleet again, starts at
|
|
193
|
+ |
round zero, and records the job it replays, so a reproducibility check never
|
|
194
|
+ |
reuses the original job's checkpoints.
|
|
195
|
+ |
"""
|
|
196
|
+ |
@spec replay(User.t(), String.t(), map()) :: {:ok, Job.t()} | {:error, term()}
|
|
197
|
+ |
def replay(%User{} = user, id, attributes) when is_map(attributes) do
|
|
198
|
+ |
with :ok <- operator(user),
|
|
199
|
+ |
{:ok, job} <- fetch(id),
|
|
200
|
+ |
{:ok, conversation_id, owner_visitor_id} <- previous_surface(job, attributes) do
|
|
201
|
+ |
start(
|
|
202
|
+ |
user,
|
|
203
|
+ |
replay_attributes(job, %{
|
|
204
|
+ |
conversation_id: conversation_id,
|
|
205
|
+ |
owner_visitor_id: owner_visitor_id
|
|
206
|
+ |
})
|
|
207
|
+ |
)
|
|
208
|
+ |
end
|
|
209
|
+ |
end
|
|
210
|
+ |
|
|
211
|
+ |
@doc "Records one append-only receipt for a job."
|
|
212
|
+ |
@spec record_receipt(Job.t(), String.t(), map()) :: {:ok, Receipt.t()} | {:error, term()}
|
|
213
|
+ |
def record_receipt(%Job{} = job, kind, payload) when is_binary(kind) and is_map(payload) do
|
|
214
|
+ |
sequence = Repo.aggregate(from(r in Receipt, where: r.job_id == ^job.id), :count) + 1
|
|
215
|
+ |
body = Map.put(payload, "recorded_at", DateTime.to_iso8601(DateTime.utc_now()))
|
|
216
|
+ |
|
|
217
|
+ |
%Receipt{job_id: job.id}
|
|
218
|
+ |
|> Receipt.changeset(%{
|
|
219
|
+ |
kind: kind,
|
|
220
|
+ |
sequence: sequence,
|
|
221
|
+ |
receipt_ref: "continual-learning-#{kind}:#{job.id}:#{sequence}",
|
|
222
|
+ |
payload: body,
|
|
223
|
+ |
digest: Canonical.digest!(Map.put(body, "job_id", job.id))
|
|
224
|
+ |
})
|
|
225
|
+ |
|> Repo.insert()
|
|
226
|
+ |
end
|
|
227
|
+ |
|
|
228
|
+ |
@doc "The bounded evidence export for one job."
|
|
229
|
+ |
@spec export_evidence(User.t(), String.t()) :: {:ok, map()} | {:error, term()}
|
|
230
|
+ |
def export_evidence(%User{} = user, id) when is_binary(id) do
|
|
231
|
+ |
with :ok <- operator(user),
|
|
232
|
+ |
{:ok, job} <- fetch(id) do
|
|
233
|
+ |
{:ok,
|
|
234
|
+ |
%{
|
|
235
|
+ |
"schema" => "openagents.continual_learning_evidence.v1",
|
|
236
|
+ |
"exported_at" => DateTime.utc_now(),
|
|
237
|
+ |
"job" => projection(job),
|
|
238
|
+ |
"checkpoints" => Enum.map(checkpoints(job), &checkpoint_projection/1),
|
|
239
|
+ |
"receipts" => Enum.map(receipts(job), &receipt_projection/1),
|
|
240
|
+ |
"artifact" => artifact_projection(artifact(job))
|
|
241
|
+ |
}}
|
|
242
|
+ |
end
|
|
243
|
+ |
end
|
|
244
|
+ |
|
|
245
|
+ |
@doc "The ordered checkpoint chain of one job."
|
|
246
|
+ |
@spec checkpoints(Job.t()) :: [Checkpoint.t()]
|
|
247
|
+ |
def checkpoints(%Job{} = job) do
|
|
248
|
+ |
Checkpoint
|
|
249
|
+ |
|> where([checkpoint], checkpoint.job_id == ^job.id)
|
|
250
|
+ |
|> order_by([checkpoint], asc: checkpoint.round)
|
|
251
|
+ |
|> Repo.all()
|
|
252
|
+ |
end
|
|
253
|
+ |
|
|
254
|
+ |
@doc "The ordered receipts of one job."
|
|
255
|
+ |
@spec receipts(Job.t()) :: [Receipt.t()]
|
|
256
|
+ |
def receipts(%Job{} = job) do
|
|
257
|
+ |
Receipt
|
|
258
|
+ |
|> where([receipt], receipt.job_id == ^job.id)
|
|
259
|
+ |
|> order_by([receipt], asc: receipt.sequence)
|
|
260
|
+ |
|> Repo.all()
|
|
261
|
+ |
end
|
|
262
|
+ |
|
|
263
|
+ |
@doc "The terminal artifact of one job, or `nil`."
|
|
264
|
+ |
@spec artifact(Job.t()) :: Artifact.t() | nil
|
|
265
|
+ |
def artifact(%Job{} = job), do: Repo.get_by(Artifact, job_id: job.id)
|
|
266
|
+ |
|
|
267
|
+ |
@doc "The latest checkpoint of one job, or `nil`."
|
|
268
|
+ |
@spec latest_checkpoint(Job.t()) :: Checkpoint.t() | nil
|
|
269
|
+ |
def latest_checkpoint(%Job{} = job) do
|
|
270
|
+ |
Checkpoint
|
|
271
|
+ |
|> where([checkpoint], checkpoint.job_id == ^job.id)
|
|
272
|
+ |
|> order_by([checkpoint], desc: checkpoint.round)
|
|
273
|
+ |
|> limit(1)
|
|
274
|
+ |
|> Repo.one()
|
|
275
|
+ |
end
|
|
276
|
+ |
|
|
277
|
+ |
@doc "How many continual-learning jobs are queued or running right now."
|
|
278
|
+ |
@spec active_count() :: non_neg_integer()
|
|
279
|
+ |
def active_count do
|
|
280
|
+ |
Repo.aggregate(from(job in Job, where: job.status in ^@active_statuses), :count)
|
|
281
|
+ |
end
|
|
282
|
+ |
|
|
283
|
+ |
@doc "Reloads one job by id."
|
|
284
|
+ |
@spec fetch(String.t()) :: {:ok, Job.t()} | {:error, :not_found}
|
|
285
|
+ |
def fetch(id) when is_binary(id) do
|
|
286
|
+ |
case Ecto.UUID.cast(id) do
|
|
287
|
+ |
{:ok, uuid} ->
|
|
288
|
+ |
case Repo.get(Job, uuid) do
|
|
289
|
+ |
nil -> {:error, :not_found}
|
|
290
|
+ |
job -> {:ok, job}
|
|
291
|
+ |
end
|
|
292
|
+ |
|
|
293
|
+ |
:error ->
|
|
294
|
+ |
{:error, :not_found}
|
|
295
|
+ |
end
|
|
296
|
+ |
end
|
|
297
|
+ |
|
|
298
|
+ |
@doc "Moves a job's lifecycle fields."
|
|
299
|
+ |
@spec update_lifecycle(Job.t(), map()) :: {:ok, Job.t()} | {:error, term()}
|
|
300
|
+ |
def update_lifecycle(%Job{} = job, attributes) when is_map(attributes) do
|
|
301
|
+ |
job
|
|
302
|
+ |
|> Job.lifecycle_changeset(attributes)
|
|
303
|
+ |
|> Repo.update()
|
|
304
|
+ |
end
|
|
305
|
+ |
|
|
306
|
+ |
@doc """
|
|
307
|
+ |
Terminalizes a job once. An already-terminal job is returned unchanged, so a
|
|
308
|
+ |
cancel racing the round loop cannot rewrite the first terminal state.
|
|
309
|
+ |
"""
|
|
310
|
+ |
@spec terminalize(Job.t(), String.t(), String.t() | nil) :: {:ok, Job.t()} | {:error, term()}
|
|
311
|
+ |
def terminalize(%Job{} = job, status, error_code) do
|
|
312
|
+ |
Repo.transaction(fn ->
|
|
313
|
+ |
locked =
|
|
314
|
+ |
Job
|
|
315
|
+ |
|> where([row], row.id == ^job.id)
|
|
316
|
+ |
|> lock("FOR UPDATE")
|
|
317
|
+ |
|> Repo.one()
|
|
318
|
+ |
|
|
319
|
+ |
cond do
|
|
320
|
+ |
is_nil(locked) ->
|
|
321
|
+ |
Repo.rollback(:not_found)
|
|
322
|
+ |
|
|
323
|
+ |
Job.terminal?(locked) ->
|
|
324
|
+ |
locked
|
|
325
|
+ |
|
|
326
|
+ |
true ->
|
|
327
|
+ |
locked
|
|
328
|
+ |
|> Job.lifecycle_changeset(%{
|
|
329
|
+ |
status: status,
|
|
330
|
+ |
error_code: error_code,
|
|
331
|
+ |
completed_at: DateTime.utc_now()
|
|
332
|
+ |
})
|
|
333
|
+ |
|> Repo.update()
|
|
334
|
+ |
|> case do
|
|
335
|
+ |
{:ok, updated} -> updated
|
|
336
|
+ |
{:error, reason} -> Repo.rollback(reason)
|
|
337
|
+ |
end
|
|
338
|
+ |
end
|
|
339
|
+ |
end)
|
|
340
|
+ |
end
|
|
341
|
+ |
|
|
342
|
+ |
@doc "The public projection of one job."
|
|
343
|
+ |
@spec projection(Job.t()) :: map()
|
|
344
|
+ |
def projection(%Job{} = job) do
|
|
345
|
+ |
%{
|
|
346
|
+ |
"id" => job.id,
|
|
347
|
+ |
"buyer_ref" => job.buyer_ref,
|
|
348
|
+ |
"buyer_class" => job.buyer_class,
|
|
349
|
+ |
"objective" => job.objective,
|
|
350
|
+ |
"objective_version" => job.objective_version,
|
|
351
|
+ |
"base_model_ref" => job.base_model_ref,
|
|
352
|
+ |
"base_model_digest" => job.base_model_digest,
|
|
353
|
+ |
"training_code_digest" => job.training_code_digest,
|
|
354
|
+ |
"configuration_digest" => job.configuration_digest,
|
|
355
|
+ |
"datasets" => job.datasets,
|
|
356
|
+ |
"evaluation" => job.evaluation,
|
|
357
|
+ |
"budget" => job.budget,
|
|
358
|
+ |
"runtime_class" => job.runtime_class,
|
|
359
|
+ |
"capacity_receipt" => job.capacity_receipt,
|
|
360
|
+ |
"stopping_policy" => job.stopping_policy,
|
|
361
|
+ |
"admission_digest" => job.admission_digest,
|
|
362
|
+ |
"status" => job.status,
|
|
363
|
+ |
"error_code" => job.error_code,
|
|
364
|
+ |
"rounds_completed" => job.rounds_completed,
|
|
365
|
+ |
"resume_count" => job.resume_count,
|
|
366
|
+ |
"usage" => job.usage,
|
|
367
|
+ |
"work_job_id" => job.work_job_id,
|
|
368
|
+ |
"replay_of_id" => job.replay_of_id,
|
|
369
|
+ |
"started_at" => job.started_at,
|
|
370
|
+ |
"completed_at" => job.completed_at
|
|
371
|
+ |
}
|
|
372
|
+ |
end
|
|
373
|
+ |
|
|
374
|
+ |
# ── dataset admission ──────────────────────────────────────────────────────
|
|
375
|
+ |
|
|
376
|
+ |
@doc """
|
|
377
|
+ |
Resolves one licensed dataset reference into its exact binding.
|
|
378
|
+ |
|
|
379
|
+ |
The listing must be available, licensed to the job's buyer class, licensed
|
|
380
|
+ |
for the requested use, and licensed for the custody the runtime class
|
|
381
|
+ |
provides, and the buyer must already hold an admitted acceptance receipt.
|
|
382
|
+ |
"""
|
|
383
|
+ |
@spec bind_dataset(map(), String.t(), String.t(), String.t(), String.t()) ::
|
|
384
|
+ |
{:ok, map()} | {:error, term()}
|
|
385
|
+ |
def bind_dataset(reference, purpose, buyer_ref, buyer_class, runtime_class)
|
|
386
|
+ |
when is_map(reference) do
|
|
387
|
+ |
with {:ok, listing_id} <- reference_field(reference, "listing_id"),
|
|
388
|
+ |
{:ok, acceptance_ref} <- reference_field(reference, "acceptance_ref"),
|
|
389
|
+ |
{:ok, access} <- authorize(listing_id, purpose, buyer_ref, acceptance_ref),
|
|
390
|
+ |
{:ok, listing} <- available_listing(listing_id),
|
|
391
|
+ |
:ok <- licensed_buyer_class(listing, buyer_class),
|
|
392
|
+ |
:ok <- licensed_use(listing, purpose),
|
|
393
|
+ |
:ok <- licensed_custody(listing, runtime_class) do
|
|
394
|
+ |
{:ok,
|
|
395
|
+ |
%{
|
|
396
|
+ |
"listing_id" => listing.id,
|
|
397
|
+ |
"acceptance_ref" => acceptance_ref,
|
|
398
|
+ |
"purpose" => purpose,
|
|
399
|
+ |
"source_ref_digest" => Canonical.sha256(access.source_ref),
|
|
400
|
+ |
"artifact_digest" => listing.artifact_digest,
|
|
401
|
+ |
"provenance_digest" => listing.provenance_digest,
|
|
402
|
+ |
"license_digest" => listing.license_digest,
|
|
403
|
+ |
"listing_digest" => listing.listing_digest,
|
|
404
|
+ |
"license_contract_ref" => listing.license_contract_ref,
|
|
405
|
+ |
"license_expires_at" => DateTime.to_iso8601(listing.license_expires_at),
|
|
406
|
+ |
"record_count" => listing.record_count
|
|
407
|
+ |
}}
|
|
408
|
+ |
end
|
|
409
|
+ |
end
|
|
410
|
+ |
|
|
411
|
+ |
defp datasets(attributes, buyer_ref, buyer_class, runtime_class) do
|
|
412
|
+ |
references = Map.get(attributes, :datasets)
|
|
413
|
+ |
|
|
414
|
+ |
cond do
|
|
415
|
+ |
not is_list(references) or references == [] ->
|
|
416
|
+ |
{:error, :datasets_required}
|
|
417
|
+ |
|
|
418
|
+ |
length(references) > Bounds.maximum_datasets() ->
|
|
419
|
+ |
{:error, :too_many_datasets}
|
|
420
|
+ |
|
|
421
|
+ |
true ->
|
|
422
|
+ |
bind_all(references, @training_purpose, buyer_ref, buyer_class, runtime_class)
|
|
423
|
+ |
end
|
|
424
|
+ |
end
|
|
425
|
+ |
|
|
426
|
+ |
defp bind_all(references, purpose, buyer_ref, buyer_class, runtime_class) do
|
|
427
|
+ |
Enum.reduce_while(references, {:ok, []}, fn reference, {:ok, bound} ->
|
|
428
|
+ |
case bind_dataset(reference, purpose, buyer_ref, buyer_class, runtime_class) do
|
|
429
|
+ |
{:ok, binding} -> {:cont, {:ok, bound ++ [binding]}}
|
|
430
|
+ |
{:error, reason} -> {:halt, {:error, reason}}
|
|
431
|
+ |
end
|
|
432
|
+ |
end)
|
|
433
|
+ |
end
|
|
434
|
+ |
|
|
435
|
+ |
defp authorize(listing_id, purpose, buyer_ref, acceptance_ref) do
|
|
436
|
+ |
case ArtifactCatalog.authorize_source_access(listing_id, %{
|
|
437
|
+ |
purpose: purpose,
|
|
438
|
+ |
buyer_ref: buyer_ref,
|
|
439
|
+ |
acceptance_ref: acceptance_ref
|
|
440
|
+ |
}) do
|
|
441
|
+ |
{:ok, access} ->
|
|
442
|
+ |
{:ok, access}
|
|
443
|
+ |
|
|
444
|
+ |
# The catalog distinguishes a listing that is gone from one whose license
|
|
445
|
+ |
# window closed, and the refusal has to keep that distinction.
|
|
446
|
+ |
{:error, reason} when reason in [:not_found, :listing_removed, :stale_license] ->
|
|
447
|
+ |
{:error, {:dataset_unavailable, reason}}
|
|
448
|
+ |
|
|
449
|
+ |
{:error, reason} ->
|
|
450
|
+ |
{:error, {:dataset_not_authorized, reason}}
|
|
451
|
+ |
end
|
|
452
|
+ |
end
|
|
453
|
+ |
|
|
454
|
+ |
defp available_listing(listing_id) do
|
|
455
|
+ |
case ArtifactCatalog.get_public_listing(listing_id) do
|
|
456
|
+ |
{:ok, listing} -> {:ok, listing}
|
|
457
|
+ |
{:error, reason} -> {:error, {:dataset_unavailable, reason}}
|
|
458
|
+ |
end
|
|
459
|
+ |
end
|
|
460
|
+ |
|
|
461
|
+ |
defp licensed_buyer_class(listing, buyer_class) do
|
|
462
|
+ |
if listing.buyer_class == buyer_class,
|
|
463
|
+ |
do: :ok,
|
|
464
|
+ |
else: {:error, {:dataset_buyer_class_mismatch, listing.id}}
|
|
465
|
+ |
end
|
|
466
|
+ |
|
|
467
|
+ |
defp licensed_use(listing, purpose) do
|
|
468
|
+ |
terms = listing.license_terms || %{}
|
|
469
|
+ |
allowed = List.wrap(terms["allowed_uses"])
|
|
470
|
+ |
use_name = if purpose == @evaluation_purpose, do: "evaluation", else: "training"
|
|
471
|
+ |
|
|
472
|
+ |
cond do
|
|
473
|
+ |
terms["opt_in"] != true -> {:error, {:consent_missing, listing.id}}
|
|
474
|
+ |
use_name not in allowed -> {:error, {:use_not_licensed, listing.id, use_name}}
|
|
475
|
+ |
true -> :ok
|
|
476
|
+ |
end
|
|
477
|
+ |
end
|
|
478
|
+ |
|
|
479
|
+ |
defp licensed_custody(listing, runtime_class) do
|
|
480
|
+ |
location = data_location(runtime_class)
|
|
481
|
+ |
licensed = List.wrap((listing.license_terms || %{})["data_locations"])
|
|
482
|
+ |
|
|
483
|
+ |
cond do
|
|
484
|
+ |
location not in Bounds.admitted_custody() ->
|
|
485
|
+ |
{:error, {:unsupported_custody, location}}
|
|
486
|
+ |
|
|
487
|
+ |
licensed != [] and location not in licensed ->
|
|
488
|
+ |
{:error, {:unsupported_custody, listing.id}}
|
|
489
|
+ |
|
|
490
|
+ |
true ->
|
|
491
|
+ |
:ok
|
|
492
|
+ |
end
|
|
493
|
+ |
end
|
|
494
|
+ |
|
|
495
|
+ |
defp reverify_datasets(%Job{} = job) do
|
|
496
|
+ |
bindings = job.datasets ++ List.wrap(get_in(job.evaluation, ["corpus"]))
|
|
497
|
+ |
|
|
498
|
+ |
Enum.reduce_while(bindings, :ok, fn binding, :ok ->
|
|
499
|
+ |
case bind_dataset(
|
|
500
|
+ |
binding,
|
|
501
|
+ |
binding["purpose"],
|
|
502
|
+ |
job.buyer_ref,
|
|
503
|
+ |
job.buyer_class,
|
|
504
|
+ |
job.runtime_class
|
|
505
|
+ |
) do
|
|
506
|
+ |
{:ok, rebound} ->
|
|
507
|
+ |
if rebound["license_digest"] == binding["license_digest"] and
|
|
508
|
+ |
rebound["artifact_digest"] == binding["artifact_digest"] do
|
|
509
|
+ |
{:cont, :ok}
|
|
510
|
+ |
else
|
|
511
|
+ |
{:halt, {:error, {:dataset_moved, binding["listing_id"]}}}
|
|
512
|
+ |
end
|
|
513
|
+ |
|
|
514
|
+ |
{:error, reason} ->
|
|
515
|
+ |
{:halt, {:error, reason}}
|
|
516
|
+ |
end
|
|
517
|
+ |
end)
|
|
518
|
+ |
end
|
|
519
|
+ |
|
|
520
|
+ |
# ── evaluation admission ───────────────────────────────────────────────────
|
|
521
|
+ |
|
|
522
|
+ |
defp evaluation(attributes, buyer_ref, buyer_class, runtime_class) do
|
|
523
|
+ |
case Map.get(attributes, :evaluation) do
|
|
524
|
+ |
evaluation when is_map(evaluation) ->
|
|
525
|
+ |
admit_evaluation(evaluation, buyer_ref, buyer_class, runtime_class)
|
|
526
|
+ |
|
|
527
|
+ |
_missing ->
|
|
528
|
+ |
{:error, :evaluation_required}
|
|
529
|
+ |
end
|
|
530
|
+ |
end
|
|
531
|
+ |
|
|
532
|
+ |
defp admit_evaluation(evaluation, buyer_ref, buyer_class, runtime_class) do
|
|
533
|
+ |
with {:ok, corpus} <-
|
|
534
|
+ |
corpus(evaluation, buyer_ref, buyer_class, runtime_class),
|
|
535
|
+ |
{:ok, verifier} <- verifier(evaluation),
|
|
536
|
+ |
{:ok, criteria} <- acceptance_criteria(evaluation),
|
|
537
|
+ |
{:ok, target} <- target_metric(evaluation) do
|
|
538
|
+ |
{:ok,
|
|
539
|
+ |
%{
|
|
540
|
+ |
"corpus" => corpus,
|
|
541
|
+ |
"corpus_digest" => Canonical.digest!(Enum.map(corpus, & &1["artifact_digest"])),
|
|
542
|
+ |
"verifier" => verifier,
|
|
543
|
+ |
"separation_required" => evaluation[:separation_required] == true,
|
|
544
|
+ |
"acceptance_criteria" => criteria,
|
|
545
|
+ |
"target_metric" => target.metric,
|
|
546
|
+ |
"target_value" => target.value,
|
|
547
|
+ |
"policy_version" => Map.get(evaluation, :policy_version, 1)
|
|
548
|
+ |
}}
|
|
549
|
+ |
end
|
|
550
|
+ |
end
|
|
551
|
+ |
|
|
552
|
+ |
defp corpus(evaluation, buyer_ref, buyer_class, runtime_class) do
|
|
553
|
+ |
references = Map.get(evaluation, :corpus)
|
|
554
|
+ |
|
|
555
|
+ |
cond do
|
|
556
|
+ |
not is_list(references) or references == [] ->
|
|
557
|
+ |
{:error, :evaluation_corpus_required}
|
|
558
|
+ |
|
|
559
|
+ |
length(references) > Bounds.maximum_datasets() ->
|
|
560
|
+ |
{:error, :too_many_datasets}
|
|
561
|
+ |
|
|
562
|
+ |
true ->
|
|
563
|
+ |
bind_all(references, @evaluation_purpose, buyer_ref, buyer_class, runtime_class)
|
|
564
|
+ |
end
|
|
565
|
+ |
end
|
|
566
|
+ |
|
|
567
|
+ |
defp verifier(evaluation) do
|
|
568
|
+ |
verifier = Map.get(evaluation, :verifier)
|
|
569
|
+ |
separation = evaluation[:separation_required] == true
|
|
570
|
+ |
|
|
571
|
+ |
cond do
|
|
572
|
+ |
not is_map(verifier) or not is_binary(verifier[:id]) ->
|
|
573
|
+ |
{:error, :evaluator_required}
|
|
574
|
+ |
|
|
575
|
+ |
verifier[:admitted] != true ->
|
|
576
|
+ |
{:error, :evaluator_not_admitted}
|
|
577
|
+ |
|
|
578
|
+ |
separation and verifier[:independent_of_producer] != true ->
|
|
579
|
+ |
{:error, :evaluator_not_independent}
|
|
580
|
+ |
|
|
581
|
+ |
true ->
|
|
582
|
+ |
{:ok,
|
|
583
|
+ |
%{
|
|
584
|
+ |
"id" => verifier[:id],
|
|
585
|
+ |
"admitted" => true,
|
|
586
|
+ |
"independent_of_producer" => verifier[:independent_of_producer] == true,
|
|
587
|
+ |
"policy_digest" => Canonical.digest!(%{"verifier" => verifier[:id]})
|
|
588
|
+ |
}}
|
|
589
|
+ |
end
|
|
590
|
+ |
end
|
|
591
|
+ |
|
|
592
|
+ |
defp acceptance_criteria(evaluation) do
|
|
593
|
+ |
criteria = List.wrap(Map.get(evaluation, :acceptance_criteria))
|
|
594
|
+ |
|
|
595
|
+ |
if criteria != [] and Enum.all?(criteria, &(is_binary(&1) and String.trim(&1) != "")) do
|
|
596
|
+ |
{:ok, criteria}
|
|
597
|
+ |
else
|
|
598
|
+ |
{:error, :acceptance_criteria_required}
|
|
599
|
+ |
end
|
|
600
|
+ |
end
|
|
601
|
+ |
|
|
602
|
+ |
defp target_metric(evaluation) do
|
|
603
|
+ |
metric = Map.get(evaluation, :target_metric)
|
|
604
|
+ |
value = Map.get(evaluation, :target_value)
|
|
605
|
+ |
|
|
606
|
+ |
if is_binary(metric) and metric != "" and is_number(value) do
|
|
607
|
+ |
{:ok, %{metric: metric, value: value}}
|
|
608
|
+ |
else
|
|
609
|
+ |
{:error, :evaluation_target_required}
|
|
610
|
+ |
end
|
|
611
|
+ |
end
|
|
612
|
+ |
|
|
613
|
+ |
# ── other admission checks ─────────────────────────────────────────────────
|
|
614
|
+ |
|
|
615
|
+ |
defp feature_enabled do
|
|
616
|
+ |
if Bounds.enabled?(), do: :ok, else: {:error, :continual_learning_disabled}
|
|
617
|
+ |
end
|
|
618
|
+ |
|
|
619
|
+ |
defp operator(user) do
|
|
620
|
+ |
if Accounts.admin?(user), do: :ok, else: {:error, :operator_required}
|
|
621
|
+ |
end
|
|
622
|
+ |
|
|
623
|
+ |
defp buyer_ref(attributes) do
|
|
624
|
+ |
admitted = Bounds.buyer_ref()
|
|
625
|
+ |
requested = Map.get(attributes, :buyer_ref)
|
|
626
|
+ |
|
|
627
|
+ |
cond do
|
|
628
|
+ |
not is_binary(admitted) or admitted == "" -> {:error, :buyer_not_configured}
|
|
629
|
+ |
requested != admitted -> {:error, :buyer_not_admitted}
|
|
630
|
+ |
true -> {:ok, admitted}
|
|
631
|
+ |
end
|
|
632
|
+ |
end
|
|
633
|
+ |
|
|
634
|
+ |
defp buyer_class do
|
|
635
|
+ |
case Bounds.buyer_class() do
|
|
636
|
+ |
value when is_binary(value) and value != "" -> {:ok, value}
|
|
637
|
+ |
_missing -> {:error, :buyer_not_configured}
|
|
638
|
+ |
end
|
|
639
|
+ |
end
|
|
640
|
+ |
|
|
641
|
+ |
defp objective(attributes) do
|
|
642
|
+ |
case Map.get(attributes, :objective) do
|
|
643
|
+ |
value when is_binary(value) ->
|
|
644
|
+ |
trimmed = String.trim(value)
|
|
645
|
+ |
|
|
646
|
+ |
if trimmed != "" and byte_size(trimmed) <= 2_000,
|
|
647
|
+ |
do: {:ok, trimmed},
|
|
648
|
+ |
else: {:error, :objective_invalid}
|
|
649
|
+ |
|
|
650
|
+ |
_missing ->
|
|
651
|
+ |
{:error, :objective_invalid}
|
|
652
|
+ |
end
|
|
653
|
+ |
end
|
|
654
|
+ |
|
|
655
|
+ |
defp objective_version(attributes) do
|
|
656
|
+ |
case Map.get(attributes, :objective_version) do
|
|
657
|
+ |
value when is_integer(value) and value > 0 -> {:ok, value}
|
|
658
|
+ |
_invalid -> {:error, :objective_version_invalid}
|
|
659
|
+ |
end
|
|
660
|
+ |
end
|
|
661
|
+ |
|
|
662
|
+ |
defp base_model(attributes) do
|
|
663
|
+ |
admitted = Bounds.admitted_base_models()
|
|
664
|
+ |
requested = Map.get(attributes, :base_model_ref)
|
|
665
|
+ |
digest = Map.get(attributes, :base_model_digest)
|
|
666
|
+ |
|
|
667
|
+ |
case Map.fetch(admitted, requested) do
|
|
668
|
+ |
{:ok, admitted_digest} when is_binary(digest) and digest != admitted_digest ->
|
|
669
|
+ |
{:error, :base_model_digest_mismatch}
|
|
670
|
+ |
|
|
671
|
+ |
{:ok, admitted_digest} ->
|
|
672
|
+ |
{:ok, %{ref: requested, digest: admitted_digest}}
|
|
673
|
+ |
|
|
674
|
+ |
:error ->
|
|
675
|
+ |
{:error, :base_model_not_admitted}
|
|
676
|
+ |
end
|
|
677
|
+ |
end
|
|
678
|
+ |
|
|
679
|
+ |
defp training_code_digest do
|
|
680
|
+ |
case Bounds.training_code_digest() do
|
|
681
|
+ |
value when is_binary(value) -> {:ok, value}
|
|
682
|
+ |
_missing -> {:error, :training_code_not_pinned}
|
|
683
|
+ |
end
|
|
684
|
+ |
end
|
|
685
|
+ |
|
|
686
|
+ |
defp configuration(attributes) do
|
|
687
|
+ |
case Map.get(attributes, :configuration, %{}) do
|
|
688
|
+ |
value when is_map(value) ->
|
|
689
|
+ |
if byte_size(Jason.encode!(value)) <= 8_192,
|
|
690
|
+ |
do: {:ok, value},
|
|
691
|
+ |
else: {:error, :configuration_too_large}
|
|
692
|
+ |
|
|
693
|
+ |
_invalid ->
|
|
694
|
+ |
{:error, :configuration_invalid}
|
|
695
|
+ |
end
|
|
696
|
+ |
end
|
|
697
|
+ |
|
|
698
|
+ |
defp runtime_class(attributes) do
|
|
699
|
+ |
requested = Map.get(attributes, :runtime_class)
|
|
700
|
+ |
|
|
701
|
+ |
if is_binary(requested) and requested in Bounds.runtime_classes(),
|
|
702
|
+ |
do: {:ok, requested},
|
|
703
|
+ |
else: {:error, :runtime_class_not_admitted}
|
|
704
|
+ |
end
|
|
705
|
+ |
|
|
706
|
+ |
defp budget(attributes) do
|
|
707
|
+ |
case Map.get(attributes, :budget) do
|
|
708
|
+ |
%{} = budget ->
|
|
709
|
+ |
amount = budget[:usd_cents] || budget["usd_cents"]
|
|
710
|
+ |
|
|
711
|
+ |
if is_integer(amount) and amount > 0,
|
|
712
|
+ |
do: {:ok, %{"unit" => "usd_cents", "amount" => amount}},
|
|
713
|
+ |
else: {:error, :budget_invalid}
|
|
714
|
+ |
|
|
715
|
+ |
_missing ->
|
|
716
|
+ |
{:error, :budget_invalid}
|
|
717
|
+ |
end
|
|
718
|
+ |
end
|
|
719
|
+ |
|
|
720
|
+ |
defp stopping_policy(attributes) do
|
|
721
|
+ |
policy = Map.get(attributes, :stopping_policy)
|
|
722
|
+ |
rounds = is_map(policy) && (policy[:maximum_rounds] || policy["maximum_rounds"])
|
|
723
|
+ |
|
|
724
|
+ |
cond do
|
|
725
|
+ |
not is_map(policy) ->
|
|
726
|
+ |
{:error, :stopping_policy_required}
|
|
727
|
+ |
|
|
728
|
+ |
not (is_integer(rounds) and rounds > 0) ->
|
|
729
|
+ |
{:error, :stopping_policy_required}
|
|
730
|
+ |
|
|
731
|
+ |
rounds > Bounds.maximum_rounds() ->
|
|
732
|
+ |
{:error, :stopping_policy_exceeds_bound}
|
|
733
|
+ |
|
|
734
|
+ |
true ->
|
|
735
|
+ |
minimum_improvement = policy[:minimum_improvement] || policy["minimum_improvement"] || 0.0
|
|
736
|
+ |
|
|
737
|
+ |
{:ok,
|
|
738
|
+ |
%{
|
|
739
|
+ |
"maximum_rounds" => rounds,
|
|
740
|
+ |
"minimum_improvement" => minimum_improvement,
|
|
741
|
+ |
"wall_clock_ms" => Bounds.wall_clock_ms()
|
|
742
|
+ |
}}
|
|
743
|
+ |
end
|
|
744
|
+ |
end
|
|
745
|
+ |
|
|
746
|
+ |
defp concurrency do
|
|
747
|
+ |
if active_count() < Bounds.concurrency_limit(),
|
|
748
|
+ |
do: :ok,
|
|
749
|
+ |
else: {:error, :continual_learning_at_capacity}
|
|
750
|
+ |
end
|
|
751
|
+ |
|
|
752
|
+ |
defp capacity(user, runtime_class, budget, stopping_policy) do
|
|
753
|
+ |
requirement = %{
|
|
754
|
+ |
"quantity" => 1,
|
|
755
|
+ |
"isolation" => isolation(runtime_class),
|
|
756
|
+ |
"egress" => "policy_broker",
|
|
757
|
+ |
"data_location" => data_location(runtime_class),
|
|
758
|
+ |
"target" => "openagents_managed",
|
|
759
|
+ |
"tools" => ["shell"],
|
|
760
|
+ |
"duration_seconds" => duration_seconds(stopping_policy),
|
|
761
|
+ |
"budget" => %{"currency" => "usd_cents", "amount" => budget["amount"]}
|
|
762
|
+ |
}
|
|
763
|
+ |
|
|
764
|
+ |
case Capacity.match(user, requirement) do
|
|
765
|
+ |
{:ok, match} ->
|
|
766
|
+ |
candidate = Enum.find(match["candidates"], &(&1["class"] == runtime_class))
|
|
767
|
+ |
|
|
768
|
+ |
if candidate do
|
|
769
|
+ |
{:ok,
|
|
770
|
+ |
%{
|
|
771
|
+ |
"schema" => match["schema"],
|
|
772
|
+ |
"matched_at" => match["generated_at"],
|
|
773
|
+ |
"requirement" => match["requirement"],
|
|
774
|
+ |
"class" => candidate["class"],
|
|
775
|
+ |
"rank" => candidate["rank"],
|
|
776
|
+ |
"evidence" => candidate["evidence"],
|
|
777
|
+ |
"estimate" => candidate["estimate"]
|
|
778
|
+ |
}}
|
|
779
|
+ |
else
|
|
780
|
+ |
{:error, {:capacity_unavailable, runtime_class}}
|
|
781
|
+ |
end
|
|
782
|
+ |
|
|
783
|
+ |
{:error, %{"error" => %{"code" => code}}} ->
|
|
784
|
+ |
{:error, {:capacity_unavailable, code}}
|
|
785
|
+ |
end
|
|
786
|
+ |
end
|
|
787
|
+ |
|
|
788
|
+ |
defp isolation("strong"), do: "managed_strong"
|
|
789
|
+ |
defp isolation(_class), do: "managed_standard"
|
|
790
|
+ |
|
|
791
|
+ |
defp data_location(_class), do: "openagents_managed"
|
|
792
|
+ |
|
|
793
|
+ |
defp duration_seconds(stopping_policy) do
|
|
794
|
+ |
stopping_policy
|
|
795
|
+ |
|> Map.get("wall_clock_ms", Bounds.wall_clock_ms())
|
|
796
|
+ |
|> div(1_000)
|
|
797
|
+ |
|> max(1)
|
|
798
|
+ |
end
|
|
799
|
+ |
|
|
800
|
+ |
defp identifier(attributes, key) do
|
|
801
|
+ |
case Map.get(attributes, key) do
|
|
802
|
+ |
value when is_binary(value) and value != "" -> {:ok, value}
|
|
803
|
+ |
_missing -> {:error, :"#{key}_required"}
|
|
804
|
+ |
end
|
|
805
|
+ |
end
|
|
806
|
+ |
|
|
807
|
+ |
# A dataset reference arrives either from the JSON API (string keys) or from
|
|
808
|
+ |
# an internal caller (atom keys); both name the same admitted listing.
|
|
809
|
+ |
defp reference_field(reference, "listing_id"),
|
|
810
|
+ |
do: reference_value(reference, "listing_id", :listing_id)
|
|
811
|
+ |
|
|
812
|
+ |
defp reference_field(reference, "acceptance_ref"),
|
|
813
|
+ |
do: reference_value(reference, "acceptance_ref", :acceptance_ref)
|
|
814
|
+ |
|
|
815
|
+ |
defp reference_value(reference, string_key, atom_key) do
|
|
816
|
+ |
case Map.get(reference, string_key) || Map.get(reference, atom_key) do
|
|
817
|
+ |
value when is_binary(value) and value != "" -> {:ok, value}
|
|
818
|
+ |
_missing -> {:error, {:dataset_reference_invalid, string_key}}
|
|
819
|
+ |
end
|
|
820
|
+ |
end
|
|
821
|
+ |
|
|
822
|
+ |
# ── insertion and launch ───────────────────────────────────────────────────
|
|
823
|
+ |
|
|
824
|
+ |
defp insert_job(admission) do
|
|
825
|
+ |
digest =
|
|
826
|
+ |
Canonical.digest!(%{
|
|
827
|
+ |
"buyer_ref" => admission.buyer_ref,
|
|
828
|
+ |
"objective" => admission.objective,
|
|
829
|
+ |
"objective_version" => admission.objective_version,
|
|
830
|
+ |
"base_model_digest" => admission.base_model_digest,
|
|
831
|
+ |
"training_code_digest" => admission.training_code_digest,
|
|
832
|
+ |
"configuration_digest" => admission.configuration_digest,
|
|
833
|
+ |
"dataset_digests" => Enum.map(admission.datasets, & &1["artifact_digest"]),
|
|
834
|
+ |
"license_digests" => Enum.map(admission.datasets, & &1["license_digest"]),
|
|
835
|
+ |
"evaluation_corpus_digest" => admission.evaluation["corpus_digest"],
|
|
836
|
+ |
"verifier_policy_digest" => admission.evaluation["verifier"]["policy_digest"],
|
|
837
|
+ |
"runtime_class" => admission.runtime_class,
|
|
838
|
+ |
"stopping_policy" => admission.stopping_policy,
|
|
839
|
+ |
"budget" => admission.budget
|
|
840
|
+ |
})
|
|
841
|
+ |
|
|
842
|
+ |
Repo.transaction(fn ->
|
|
843
|
+ |
changeset =
|
|
844
|
+ |
Job.admission_changeset(%Job{}, Map.put(admission, :admission_digest, digest))
|
|
845
|
+ |
|
|
846
|
+ |
with {:ok, job} <- Repo.insert(changeset),
|
|
847
|
+ |
{:ok, _receipt} <-
|
|
848
|
+ |
record_receipt(job, "admission", %{
|
|
849
|
+ |
"admission_digest" => job.admission_digest,
|
|
850
|
+ |
"buyer_ref" => job.buyer_ref,
|
|
851
|
+ |
"buyer_class" => job.buyer_class,
|
|
852
|
+ |
"objective_version" => job.objective_version,
|
|
853
|
+ |
"base_model_ref" => job.base_model_ref,
|
|
854
|
+ |
"base_model_digest" => job.base_model_digest,
|
|
855
|
+ |
"training_code_digest" => job.training_code_digest,
|
|
856
|
+ |
"configuration_digest" => job.configuration_digest,
|
|
857
|
+ |
"datasets" => job.datasets,
|
|
858
|
+ |
"evaluation" => Map.drop(job.evaluation, ["corpus"]),
|
|
859
|
+ |
"evaluation_corpus" => job.evaluation["corpus"],
|
|
860
|
+ |
"runtime_class" => job.runtime_class,
|
|
861
|
+ |
"capacity_receipt" => job.capacity_receipt,
|
|
862
|
+ |
"budget" => job.budget,
|
|
863
|
+ |
"stopping_policy" => job.stopping_policy,
|
|
864
|
+ |
"replay_of_id" => job.replay_of_id
|
|
865
|
+ |
}) do
|
|
866
|
+ |
job
|
|
867
|
+ |
else
|
|
868
|
+ |
{:error, reason} -> Repo.rollback(reason)
|
|
869
|
+ |
end
|
|
870
|
+ |
end)
|
|
871
|
+ |
end
|
|
872
|
+ |
|
|
873
|
+ |
defp launch(%Job{} = job, conversation_id, owner_visitor_id, cause) do
|
|
874
|
+ |
case Work.start_continual_learning(%{
|
|
875
|
+ |
conversation_id: conversation_id,
|
|
876
|
+ |
owner_visitor_id: owner_visitor_id,
|
|
877
|
+ |
surface: "text",
|
|
878
|
+ |
goal: job.objective,
|
|
879
|
+ |
delegation: %{
|
|
880
|
+ |
"continual_learning_job_id" => job.id,
|
|
881
|
+ |
"admission_digest" => job.admission_digest,
|
|
882
|
+ |
"cause" => cause,
|
|
883
|
+ |
"resume_count" => job.resume_count
|
|
884
|
+ |
},
|
|
885
|
+ |
authority_snapshot: %{
|
|
886
|
+ |
"buyer_ref" => job.buyer_ref,
|
|
887
|
+ |
"buyer_class" => job.buyer_class,
|
|
888
|
+ |
"runtime_class" => job.runtime_class,
|
|
889
|
+ |
"base_model_ref" => job.base_model_ref,
|
|
890
|
+ |
"base_model_digest" => job.base_model_digest,
|
|
891
|
+ |
"training_code_digest" => job.training_code_digest,
|
|
892
|
+ |
"verifier_id" => job.evaluation["verifier"]["id"]
|
|
893
|
+ |
},
|
|
894
|
+ |
budget_snapshot: Bounds.snapshot(job.runtime_class)
|
|
895
|
+ |
}) do
|
|
896
|
+ |
{:ok, work_job} ->
|
|
897
|
+ |
update_lifecycle(job, %{work_job_id: work_job.id})
|
|
898
|
+ |
|
|
899
|
+ |
{:error, reason} ->
|
|
900
|
+ |
_refusal = record_receipt(job, "refusal", %{"reason" => inspect(reason)})
|
|
901
|
+ |
_terminal = terminalize(job, "failed", "worker_start_failed")
|
|
902
|
+ |
{:error, reason}
|
|
903
|
+ |
end
|
|
904
|
+ |
end
|
|
905
|
+ |
|
|
906
|
+ |
defp resumable(%Job{} = job) do
|
|
907
|
+ |
if Job.resumable?(job), do: :ok, else: {:error, :not_resumable}
|
|
908
|
+ |
end
|
|
909
|
+ |
|
|
910
|
+ |
defp surviving_checkpoint(%Job{} = job) do
|
|
911
|
+ |
case latest_checkpoint(job) do
|
|
912
|
+ |
nil -> {:error, :checkpoint_missing}
|
|
913
|
+ |
%Checkpoint{lost: true} -> {:error, :checkpoint_lost}
|
|
914
|
+ |
%Checkpoint{} = checkpoint -> verify_chain(job, checkpoint)
|
|
915
|
+ |
end
|
|
916
|
+ |
end
|
|
917
|
+ |
|
|
918
|
+ |
defp verify_chain(%Job{} = job, %Checkpoint{} = checkpoint) do
|
|
919
|
+ |
recomputed = Canonical.digest!(checkpoint.state)
|
|
920
|
+ |
|
|
921
|
+ |
if recomputed == checkpoint.state_digest and checkpoint.round == job.rounds_completed do
|
|
922
|
+ |
{:ok, checkpoint}
|
|
923
|
+ |
else
|
|
924
|
+ |
{:error, :checkpoint_lost}
|
|
925
|
+ |
end
|
|
926
|
+ |
end
|
|
927
|
+ |
|
|
928
|
+ |
# A resume spends the admitted budget, so a job that already spent all of it
|
|
929
|
+ |
# has to be admitted again rather than resumed into the same stop.
|
|
930
|
+ |
defp budget_remaining(%Job{} = job) do
|
|
931
|
+ |
spent = Map.get(job.usage || %{}, "cost_usd_cents", 0)
|
|
932
|
+ |
amount = job.budget["amount"] || 0
|
|
933
|
+ |
|
|
934
|
+ |
if spent + Bounds.round_cost_usd_cents(job.runtime_class) <= amount,
|
|
935
|
+ |
do: :ok,
|
|
936
|
+ |
else: {:error, :budget_exhausted}
|
|
937
|
+ |
end
|
|
938
|
+ |
|
|
939
|
+ |
defp rounds_remaining(%Job{} = job) do
|
|
940
|
+ |
maximum = job.stopping_policy["maximum_rounds"] || Bounds.maximum_rounds()
|
|
941
|
+ |
if job.rounds_completed < maximum, do: :ok, else: {:error, :stopping_policy_satisfied}
|
|
942
|
+ |
end
|
|
943
|
+ |
|
|
944
|
+ |
defp previous_surface(%Job{work_job_id: nil}, attributes) do
|
|
945
|
+ |
with {:ok, conversation_id} <- identifier(attributes, :conversation_id),
|
|
946
|
+ |
{:ok, owner_visitor_id} <- identifier(attributes, :owner_visitor_id) do
|
|
947
|
+ |
{:ok, conversation_id, owner_visitor_id}
|
|
948
|
+ |
end
|
|
949
|
+ |
end
|
|
950
|
+ |
|
|
951
|
+ |
defp previous_surface(%Job{work_job_id: work_job_id}, attributes) do
|
|
952
|
+ |
case Work.get_job(work_job_id) do
|
|
953
|
+ |
nil -> previous_surface(%Job{work_job_id: nil}, attributes)
|
|
954
|
+ |
work_job -> {:ok, work_job.conversation_id, work_job.owner_visitor_id}
|
|
955
|
+ |
end
|
|
956
|
+ |
end
|
|
957
|
+ |
|
|
958
|
+ |
defp mark_resumed(%Job{} = job, %Checkpoint{} = checkpoint, capacity_receipt) do
|
|
959
|
+ |
with {:ok, _receipt} <-
|
|
960
|
+ |
record_receipt(job, "resume", %{
|
|
961
|
+ |
"from_round" => checkpoint.round,
|
|
962
|
+ |
"checkpoint_digest" => checkpoint.state_digest,
|
|
963
|
+ |
"admission_digest" => job.admission_digest,
|
|
964
|
+ |
"previous_status" => job.status,
|
|
965
|
+ |
"previous_work_job_id" => job.work_job_id,
|
|
966
|
+ |
"resume_count" => job.resume_count + 1,
|
|
967
|
+ |
"capacity_receipt" => capacity_receipt
|
|
968
|
+ |
}),
|
|
969
|
+ |
{:ok, resumed} <-
|
|
970
|
+ |
update_lifecycle(job, %{
|
|
971
|
+ |
status: "queued",
|
|
972
|
+ |
error_code: nil,
|
|
973
|
+ |
completed_at: nil,
|
|
974
|
+ |
resume_count: job.resume_count + 1,
|
|
975
|
+ |
work_job_id: nil
|
|
976
|
+ |
}) do
|
|
977
|
+ |
{:ok, resumed}
|
|
978
|
+ |
end
|
|
979
|
+ |
end
|
|
980
|
+ |
|
|
981
|
+ |
defp replay_attributes(%Job{} = job, attributes) do
|
|
982
|
+ |
%{
|
|
983
|
+ |
buyer_ref: job.buyer_ref,
|
|
984
|
+ |
objective: job.objective,
|
|
985
|
+ |
objective_version: job.objective_version,
|
|
986
|
+ |
base_model_ref: job.base_model_ref,
|
|
987
|
+ |
base_model_digest: job.base_model_digest,
|
|
988
|
+ |
configuration: job.configuration,
|
|
989
|
+ |
runtime_class: job.runtime_class,
|
|
990
|
+ |
datasets: Enum.map(job.datasets, &Map.take(&1, ["listing_id", "acceptance_ref"])),
|
|
991
|
+ |
evaluation: %{
|
|
992
|
+ |
corpus:
|
|
993
|
+ |
Enum.map(
|
|
994
|
+ |
List.wrap(job.evaluation["corpus"]),
|
|
995
|
+ |
&Map.take(&1, ["listing_id", "acceptance_ref"])
|
|
996
|
+ |
),
|
|
997
|
+ |
verifier: %{
|
|
998
|
+ |
id: job.evaluation["verifier"]["id"],
|
|
999
|
+ |
admitted: true,
|
|
1000
|
+ |
independent_of_producer: job.evaluation["verifier"]["independent_of_producer"]
|
|
1001
|
+ |
},
|
|
1002
|
+ |
separation_required: job.evaluation["separation_required"],
|
|
1003
|
+ |
acceptance_criteria: job.evaluation["acceptance_criteria"],
|
|
1004
|
+ |
target_metric: job.evaluation["target_metric"],
|
|
1005
|
+ |
target_value: job.evaluation["target_value"],
|
|
1006
|
+ |
policy_version: job.evaluation["policy_version"]
|
|
1007
|
+ |
},
|
|
1008
|
+ |
budget: %{usd_cents: job.budget["amount"]},
|
|
1009
|
+ |
stopping_policy: %{
|
|
1010
|
+ |
maximum_rounds: job.stopping_policy["maximum_rounds"],
|
|
1011
|
+ |
minimum_improvement: job.stopping_policy["minimum_improvement"]
|
|
1012
|
+ |
},
|
|
1013
|
+ |
replay_of_id: job.id,
|
|
1014
|
+ |
conversation_id: Map.get(attributes, :conversation_id),
|
|
1015
|
+ |
owner_visitor_id: Map.get(attributes, :owner_visitor_id)
|
|
1016
|
+ |
}
|
|
1017
|
+ |
end
|
|
1018
|
+ |
|
|
1019
|
+ |
# ── settlement readiness ───────────────────────────────────────────────────
|
|
1020
|
+ |
|
|
1021
|
+ |
@doc """
|
|
1022
|
+ |
The settlement-ready receipt payload for a qualified job.
|
|
1023
|
+ |
|
|
1024
|
+ |
The lane records authority and evidence, never custody: the payload names the
|
|
1025
|
+ |
buyer, the unit, the metered amount, the treasury policy that would pay it,
|
|
1026
|
+ |
and the artifact it settles, and states that no transfer happened here.
|
|
1027
|
+ |
"""
|
|
1028
|
+ |
@spec settlement_payload(Job.t(), map(), map()) :: map()
|
|
1029
|
+ |
def settlement_payload(%Job{} = job, artifact_payload, usage) do
|
|
1030
|
+ |
%{
|
|
1031
|
+ |
"settlement_policy_id" => Settlement.policy_id(),
|
|
1032
|
+ |
"unit" => Bounds.settlement_unit(),
|
|
1033
|
+ |
"buyer_ref" => job.buyer_ref,
|
|
1034
|
+ |
"buyer_class" => job.buyer_class,
|
|
1035
|
+ |
"amount" => usage["cost_usd_cents"],
|
|
1036
|
+ |
"budget" => job.budget,
|
|
1037
|
+ |
"artifact_digest" => artifact_payload["artifact_digest"],
|
|
1038
|
+ |
"accepted_outcome_state" => "accepted",
|
|
1039
|
+ |
"usage" => usage,
|
|
1040
|
+ |
"transferred" => false,
|
|
1041
|
+ |
"custody" => "no_custody_moves_in_this_lane"
|
|
1042
|
+ |
}
|
|
1043
|
+ |
end
|
|
1044
|
+ |
|
|
1045
|
+ |
@doc """
|
|
1046
|
+ |
Grades one evaluation result against the accepted-outcome contract.
|
|
1047
|
+ |
|
|
1048
|
+ |
The claim is built from the admitted evaluator policy and the job's own
|
|
1049
|
+ |
identity, so the contract, not this lane, decides whether the artifact is
|
|
1050
|
+ |
qualified.
|
|
1051
|
+ |
"""
|
|
1052
|
+ |
@spec grade(Job.t(), map(), map()) ::
|
|
1053
|
+ |
{:accepted, map()} | {:not_accepted, atom(), [term()]} | {:not_applicable, atom()}
|
|
1054
|
+ |
def grade(%Job{} = job, result, %{repository: repository, issue_number: issue_number}) do
|
|
1055
|
+ |
policy = job.evaluation
|
|
1056
|
+ |
|
|
1057
|
+ |
AcceptedOutcome.evaluate(%{
|
|
1058
|
+ |
actor: :agent,
|
|
1059
|
+ |
agents_enabled: true,
|
|
1060
|
+ |
issue: %{
|
|
1061
|
+ |
number: issue_number,
|
|
1062
|
+ |
repository: repository,
|
|
1063
|
+ |
sections: %{
|
|
1064
|
+ |
problem: job.objective,
|
|
1065
|
+ |
scope: "continual-learning job #{job.id}",
|
|
1066
|
+ |
acceptance_criteria: policy["acceptance_criteria"],
|
|
1067
|
+ |
success_metrics: "#{policy["target_metric"]} >= #{policy["target_value"]}"
|
|
1068
|
+ |
}
|
|
1069
|
+ |
},
|
|
1070
|
+ |
attempt: %{
|
|
1071
|
+ |
issue_number: issue_number,
|
|
1072
|
+ |
repository: repository,
|
|
1073
|
+ |
authority: job.buyer_ref,
|
|
1074
|
+ |
budget: job.budget,
|
|
1075
|
+ |
revision: job.admission_digest
|
|
1076
|
+ |
},
|
|
1077
|
+ |
verification: %{
|
|
1078
|
+ |
verifier: %{
|
|
1079
|
+ |
id: policy["verifier"]["id"],
|
|
1080
|
+ |
admitted: policy["verifier"]["admitted"] == true,
|
|
1081
|
+ |
independent_of_producer: policy["verifier"]["independent_of_producer"] == true
|
|
1082
|
+ |
},
|
|
1083
|
+ |
falsifier: result.falsifier,
|
|
1084
|
+ |
terminal_result: result.terminal_result,
|
|
1085
|
+ |
separation_required: policy["separation_required"] == true,
|
|
1086
|
+ |
false_green_classes: []
|
|
1087
|
+ |
},
|
|
1088
|
+ |
evidence:
|
|
1089
|
+ |
Enum.map(result.criteria, fn item ->
|
|
1090
|
+ |
%{
|
|
1091
|
+ |
criterion: item["criterion"],
|
|
1092
|
+ |
receipt: item["receipt"],
|
|
1093
|
+ |
visibility: visibility(item["visibility"])
|
|
1094
|
+ |
}
|
|
1095
|
+ |
end)
|
|
1096
|
+ |
})
|
|
1097
|
+ |
end
|
|
1098
|
+ |
|
|
1099
|
+ |
defp visibility("public"), do: :public
|
|
1100
|
+ |
defp visibility(_restricted), do: :restricted
|
|
1101
|
+ |
|
|
1102
|
+ |
# ── projections ────────────────────────────────────────────────────────────
|
|
1103
|
+ |
|
|
1104
|
+ |
defp checkpoint_projection(%Checkpoint{} = checkpoint) do
|
|
1105
|
+ |
%{
|
|
1106
|
+ |
"round" => checkpoint.round,
|
|
1107
|
+ |
"state_digest" => checkpoint.state_digest,
|
|
1108
|
+ |
"parent_digest" => checkpoint.parent_digest,
|
|
1109
|
+ |
"metrics" => checkpoint.metrics,
|
|
1110
|
+ |
"usage" => checkpoint.usage,
|
|
1111
|
+ |
"energy" => checkpoint.energy,
|
|
1112
|
+ |
"lost" => checkpoint.lost,
|
|
1113
|
+ |
"recorded_at" => checkpoint.inserted_at
|
|
1114
|
+ |
}
|
|
1115
|
+ |
end
|
|
1116
|
+ |
|
|
1117
|
+ |
defp receipt_projection(%Receipt{} = receipt) do
|
|
1118
|
+ |
%{
|
|
1119
|
+ |
"kind" => receipt.kind,
|
|
1120
|
+ |
"sequence" => receipt.sequence,
|
|
1121
|
+ |
"receipt_ref" => receipt.receipt_ref,
|
|
1122
|
+ |
"digest" => receipt.digest,
|
|
1123
|
+ |
"payload" => receipt.payload
|
|
1124
|
+ |
}
|
|
1125
|
+ |
end
|
|
1126
|
+ |
|
|
1127
|
+ |
defp artifact_projection(nil), do: nil
|
|
1128
|
+ |
|
|
1129
|
+ |
defp artifact_projection(%Artifact{} = artifact) do
|
|
1130
|
+ |
%{
|
|
1131
|
+ |
"model_ref" => artifact.model_ref,
|
|
1132
|
+ |
"model_digest" => artifact.model_digest,
|
|
1133
|
+ |
"base_model_digest" => artifact.base_model_digest,
|
|
1134
|
+ |
"training_code_digest" => artifact.training_code_digest,
|
|
1135
|
+ |
"configuration_digest" => artifact.configuration_digest,
|
|
1136
|
+ |
"dataset_bindings" => artifact.dataset_bindings,
|
|
1137
|
+ |
"checkpoint_digests" => artifact.checkpoint_digests,
|
|
1138
|
+ |
"evaluation_result" => artifact.evaluation_result,
|
|
1139
|
+ |
"accepted_outcome" => artifact.accepted_outcome,
|
|
1140
|
+ |
"settlement" => artifact.settlement,
|
|
1141
|
+ |
"artifact_digest" => artifact.artifact_digest
|
|
1142
|
+ |
}
|
|
1143
|
+ |
end
|
|
1144
|
+ |
end
|