ops/staging/validate-resilience-report.sh

58e6347eeb72 · 8 KB

#!/bin/sh
set -eu

script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
matrix="$script_dir/resilience-matrix.json"
mode=${1:-}
report=${2:-}

case "$mode" in
  --draft | --recorded | --final) ;;
  *)
    echo "usage: ops/staging/validate-resilience-report.sh [--draft|--recorded|--final] REPORT" >&2
    exit 64
    ;;
esac

if [ "$#" -ne 2 ] || [ ! -f "$report" ]; then
  echo "REPORT must be a regular file" >&2
  exit 1
fi

for command_name in jq realpath sha256sum stat; do
  if ! command -v "$command_name" >/dev/null 2>&1; then
    echo "$command_name is required to validate a staging resilience report" >&2
    exit 1
  fi
done

"$script_dir/scan-evidence.sh" "$report" >/dev/null

jq -e \
  --arg mode "$mode" \
  --slurpfile matrix "$matrix" '
  def digest: type == "string" and test("^[0-9a-f]{64}$");
  def manifest_digest: type == "string" and test("^sha256:[0-9a-f]{64}$");
  def timestamp: type == "string" and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$");
  def nonempty: type == "string" and length > 0 and length <= 500;
  def evidence_ref:
    type == "object" and
    (.path | type == "string" and test("^evidence/[A-Za-z0-9._/-]+$") and (contains("..") | not)) and
    (.sha256 | digest) and
    (.kind | nonempty);
  def valid_attempt:
    type == "object" and
    (.ordinal | type == "number" and . >= 1 and floor == .) and
    (.outcome | IN("passed", "failed", "blocked")) and
    (.started_at | timestamp) and
    (.completed_at | timestamp) and
    (.automatic_retry | type == "boolean") and
    (.explanation == null or (.explanation | nonempty)) and
    (.evidence | type == "array" and length > 0 and all(.[]; evidence_ref));
  def valid_result($expected):
    . as $result |
    ($expected | map(select(.id == $result.id)) | .[0]) as $case |
    $case != null and
    $result.title == $case.title and
    ($result.status | IN("pending", "passed", "failed", "blocked")) and
    ($result.reason == null or ($result.reason | nonempty)) and
    ($result.attempts | type == "array" and all(.[]; valid_attempt)) and
    ($result.evidence | type == "array" and all(.[]; evidence_ref)) and
    ($result.attempts as $attempts |
      [$attempts[].ordinal] == [range(1; ($attempts | length) + 1)]) and
    (if $result.status == "pending" then
       ($result.attempts | length) == 0 and $result.reason == null
     else
       ($result.attempts | length) > 0 and
       ($result.attempts[-1].outcome == $result.status) and
       ($result.evidence | length) > 0 and
       (if $result.status == "passed" then true else ($result.reason | nonempty) end)
     end);
  def issue_valid:
    (.id | nonempty) and
    (.owner | nonempty) and
    (.severity | IN("low", "medium", "high", "critical")) and
    (.disposition | IN("resolved", "accepted_non_blocking")) and
    (if .severity | IN("high", "critical") then .disposition == "resolved" else true end);

  $matrix[0].failure_injections as $expected_failures |
  $matrix[0].soak.canaries as $expected_canaries |
  . as $report |

  .schema == "openagents.staging-resilience-report.v1" and
  .matrix_revision == $matrix[0].revision and
  (.synthetic | type == "boolean") and
  (.run_id | type == "string" and test("^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")) and
  (.created_at | timestamp) and
  (.completed_at == null or (.completed_at | timestamp)) and
  .target.environment == "staging" and
  .target.base_url == "https://staging.openagents.com" and
  .target.track == "release-candidate" and
  .target.service == "openagents-staging-release" and
  (.target.project | type == "string" and test("stag"; "i")) and
  (.target.region | nonempty) and
  (.candidate.git_sha | type == "string" and test("^[0-9a-f]{40}$")) and
  (.candidate.candidate_manifest_sha256 | digest) and
  (.candidate.application_manifest_digest | manifest_digest) and
  (.candidate.application_image | endswith("@" + $report.candidate.application_manifest_digest)) and
  (.candidate.release_version | nonempty) and
  (.candidate.release_sha256 | digest) and
  (.candidate.gate14_report_sha256 | digest) and
  (.failure_injections | type == "array" and length == ($expected_failures | length)) and
  ([.failure_injections[].id] | sort) == ([$expected_failures[].id] | sort) and
  all(.failure_injections[]; valid_result($expected_failures)) and
  .soak.required_duration_seconds == $matrix[0].soak.required_duration_seconds and
  .soak.metric_sample_cadence_seconds == $matrix[0].soak.metric_sample_cadence_seconds and
  (.soak.canaries | type == "array" and length == ($expected_canaries | length)) and
  ([.soak.canaries[].id] | sort) == ([$expected_canaries[].id] | sort) and
  all(.soak.canaries[]; . as $canary |
    ($expected_canaries | map(select(.id == $canary.id)) | .[0]) as $expected |
    $canary.cadence_seconds == $expected.cadence_seconds and
    $canary.minimum_passes == $expected.minimum_passes and
    ($canary.completed_count == null or ($canary.completed_count | type == "number" and . >= 0 and floor == .)) and
    ($canary.passed_count == null or ($canary.passed_count | type == "number" and . >= 0 and floor == .)) and
    ($canary.receipt == null or ($canary.receipt | evidence_ref))) and
  (.known_issues | type == "array" and all(.[]; issue_valid)) and
  ([.. | objects | select(has("path") or has("sha256"))] | all(.[]; evidence_ref)) and
  (if $mode == "--draft" then
     .state == "draft"
   elif $mode == "--recorded" then
     .state == "recorded" and .synthetic == false and (.completed_at | timestamp) and
     all(.failure_injections[]; .status != "pending")
   else
     .state == "complete" and .synthetic == false and (.completed_at | timestamp) and
     all(.failure_injections[]; .status == "passed") and
     (.soak.started_at | timestamp) and
     (.soak.completed_at | timestamp) and
     ((.soak.completed_at | fromdateiso8601) - (.soak.started_at | fromdateiso8601) >= .soak.required_duration_seconds) and
     ((.completed_at | fromdateiso8601) >= (.soak.completed_at | fromdateiso8601)) and
     .soak.candidate_identity_stable == true and
     .soak.redeploy_count == 0 and
     (.soak.metric_sample_count | type == "number" and . >= $matrix[0].soak.minimum_metric_samples and floor == .) and
     (.soak.timeline_receipt | evidence_ref) and
     (.soak.metrics_receipt | evidence_ref) and
     all(.soak.canaries[]; . as $canary |
       ($canary.completed_count | type == "number" and . >= $canary.minimum_passes and floor == .) and
       $canary.passed_count == $canary.completed_count and
       ($canary.receipt | evidence_ref)) and
     (.soak.post_soak_smoke_receipt | evidence_ref) and
     .soak.unexplained_error_count == 0 and
     .soak.data_loss_count == 0 and
     .soak.authority_expansion_count == 0 and
     .soak.fleet_divergence_count == 0 and
     .soak.secret_leak_count == 0 and
     .soak.unexplained_restart_count == 0
   end)
' "$report" >/dev/null || {
  echo "staging resilience report does not satisfy $mode validation" >&2
  exit 1
}

report_dir=$(realpath "$(dirname -- "$report")")
refs=$(mktemp /tmp/openagents-resilience-report-refs.XXXXXX)
cleanup() {
  unlink "$refs" 2>/dev/null || true
}
trap cleanup EXIT INT TERM

jq -r '
  .. | objects |
  select(has("path") and has("sha256")) |
  [.path, .sha256] | @tsv
' "$report" >"$refs"

tab=$(printf '\t')
while IFS="$tab" read -r relative_path expected_sha256; do
  [ -n "$relative_path" ] || continue

  case "$relative_path" in
    evidence/*) ;;
    *) echo "evidence reference must stay under evidence/: $relative_path" >&2; exit 1 ;;
  esac

  case "$relative_path" in
    *..* | /* | *[!A-Za-z0-9._/-]*)
      echo "unsafe evidence reference: $relative_path" >&2
      exit 1
      ;;
  esac

  evidence_path="$report_dir/$relative_path"
  if [ ! -f "$evidence_path" ] || [ -L "$evidence_path" ]; then
    echo "missing or linked evidence file: $relative_path" >&2
    exit 1
  fi

  resolved=$(realpath "$evidence_path")
  case "$resolved" in
    "$report_dir"/evidence/*) ;;
    *) echo "evidence path escapes the report directory: $relative_path" >&2; exit 1 ;;
  esac

  actual_sha256=$(sha256sum "$evidence_path" | cut -d ' ' -f 1)
  if [ "$actual_sha256" != "$expected_sha256" ]; then
    echo "evidence checksum mismatch: $relative_path" >&2
    exit 1
  fi

  mode_bits=$(stat -c '%a' "$evidence_path" 2>/dev/null || stat -f '%Lp' "$evidence_path")
  case "$mode_bits" in
    400 | 600) ;;
    *) echo "evidence file must use mode 0400 or 0600: $relative_path" >&2; exit 1 ;;
  esac

  "$script_dir/scan-evidence.sh" "$evidence_path" >/dev/null
done <"$refs"

echo "Staging resilience report $mode validation passed."