{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/Sean-Koval/hangar/blob/main/docs/api/experiment.schema.json", "title": "hangar.experiment/v1: an experiment's index of trials", "description": "What GET /api/experiments/{id} returns: what each variant and task was pinned to, and per trial its job, the digests of the manifests it ran with, its verification and test results, an optional external grader's output, and links to its stored manifest and trace export. An index, not an analysis: effect sizes, rubrics and leaderboards belong to the sister tool. Additions within v1 are new optional properties. See docs/specs/0014-sister-tool-interfaces.md.", "type": "object", "required": ["schema", "id", "name", "state", "spec", "pins", "counts", "trials", "notes"], "properties": { "schema": {"const": "hangar.experiment/v1"}, "id": {"type": "string", "description": "The experiment's job id."}, "name": {"type": ["string", "null"]}, "state": {"enum": ["queued", "provisioning", "running", "verifying", "delivering", "succeeded", "failed", "cancelled", "awaiting_approval", "needs_input"]}, "reason": {"type": ["string", "null"]}, "requester": {"type": "string"}, "person": {"type": ["string", "null"]}, "created_at": {"type": "number"}, "finished_at": {"type": ["number", "null"]}, "spec": {"type": "object", "description": "The experiment as submitted (redacted). Its grader's canary (`grader.canary`, generated as HANGAR-CANARY- when the experiment starts if unset; #127) is masked as HANGAR-CANARY-[hidden] here and in every export.", "properties": { "variants": {"type": "object", "description": "Each variant: `{harness_lock, environment}` (#135), or the deprecated overlay.", "additionalProperties": { "type": "object", "properties": { "environment": {"type": ["string", "null"]}, "harness_lock": {"type": ["string", "null"], "pattern": "^[a-z0-9][a-z0-9._-]{0,62}@sha256:[0-9a-f]{64}$", "description": "A lock in the harness registry, by digest. Every trial is claimed from it (hangar harness thaw) and its turn 1 checked against it. Excludes every overlay field (422 otherwise). Added within v1 (#135)."}, "harness": {"type": ["string", "null"], "deprecated": true, "description": "Overlay (deprecated, #135): harness[@version]. Removed in the release after flightdeck v0's first report; use harness_lock."}, "model": {"type": ["string", "null"], "deprecated": true, "description": "Overlay (deprecated, #135): a concrete id or a policy."}, "approvals": {"enum": ["auto", "deny-risky", null], "deprecated": true, "description": "Overlay (deprecated, #135)."}, "skills": {"type": ["array", "null"], "deprecated": true, "description": "Overlay (deprecated, #135): the whole skill set, [] for none."}, "skills_remove": {"type": ["array", "null"], "deprecated": true, "description": "Overlay (deprecated, #135)."}, "instructions": {"type": ["string", "null"], "deprecated": true, "description": "Overlay (deprecated, #135)."}, "instructions_append": {"type": ["string", "null"], "deprecated": true, "description": "Overlay (deprecated, #135)."} } }},"grader": {"type": "object", "properties": {"canary": {"type": "string"}}}, "pin_sets": {"type": ["object", "null"], "additionalProperties": {"type": "string"}, "description": "Variant -> pin set id, for trials started with POST /api/trials (#178)."}, "source": {"enum": ["experiment", "shadow"], "description": "Who asked for the trials (#178; shadow: #183)."}, "task_digests": {"type": ["object", "null"], "additionalProperties": {"type": "string"}, "description": "Task id -> flightdeck's task digest for the trial cache (#180)."}, "cache_record": {"type": "boolean", "description": "Whether finished trials fill the trial cache (false for a sentinel run, #180)."}, "fork": {"type": ["object", "null"], "description": "A fork of a recorded run (docs/sessions.md#fork-at-a-step): {run, at_seq, mode, on_divergence, continuations, tainted, harness_digest, pin_set, purpose, tolerate}. Added within v1 (#179)."}}}, "pins": { "type": "object", "properties": { "variants": {"type": "object", "additionalProperties": { "type": "object", "required": ["environment", "harness", "harness_version", "model", "approvals", "skills"], "properties": { "environment": {"type": "string"}, "harness": {"type": "string"}, "harness_version": {"type": ["string", "null"], "description": "The locked version every trial ran."}, "model": {"type": ["string", "null"], "description": "The concrete model id every trial got (a policy is resolved once, at the start)."}, "model_policy": {"type": ["string", "null"]}, "model_concrete": {"type": ["boolean", "null"], "description": "The pinned model is a concrete, dated id (an experiment refuses to start otherwise); null when the harness has no model list and runs its own default. Added within v1 (#167)."}, "model_concrete_basis": {"enum": ["list", "pattern", "none", "harness_lock"], "description": "Why it counts as concrete: in the provider's cached model list (list), or, with no list cached, by its versioned shape alone (pattern; a note says so); none with no id; harness_lock when the variant's lock pins it (the lock only records concrete ids). Added within v1 (#167, #135)."}, "model_source": {"type": ["string", "null"], "description": "Where the model came from: what the model resolver reports, or harness_lock for a lock variant."}, "harness_lock": {"type": ["string", "null"], "description": "The variant's harness lock (name@sha256:...), when it runs from one. Added within v1 (#135)."}, "harness_revision": {"type": ["integer", "null"], "description": "That lock's revision. Added within v1 (#135)."}, "harness_digest": {"type": ["string", "null"], "description": "The harness every trial of the variant runs: its lock's digest, which each trial's turn 1 must match (a mismatch is drift); for an overlay variant, its first finished trial's turn-1 harness_digest (null until then). Flightdeck pin sets reference it. Added within v1 (#135)."}, "approvals": {"type": "string"}, "skills": {"type": ["array", "null"], "description": "The variant's whole skill set (null: the environment's)."}, "instructions": {"type": ["string", "null"], "description": "The digest of the instructions the variant writes, when it has an overlay (null for a harness_lock variant: the lock's instructions file is mounted)."}, "pin_set": {"type": ["string", "null"], "pattern": "^pin-[0-9a-f]{16}$", "description": "The variant's pin set (docs/api/trials.md): what its trials run against, immutable and content-addressed. A trials request names it; otherwise the experiment records one at its start. Added within v1 (#178)."} } }}, "tasks": {"type": "object", "additionalProperties": { "type": "object", "properties": { "repo": {"type": ["string", "null"]}, "base_branch": {"type": ["string", "null"]}, "base_sha": {"type": ["string", "null"], "description": "Pinned at the start from GitHub, else by the first trial's clone (then every trial checks it out)."}, "issue": {"type": ["string", "null"]}, "task_digest": {"type": ["string", "null"], "description": "The task's identity for the trial cache: sha256 of its canonical definition (instructions or issue, verify, repo, reference patch, hidden tests by digest, environment), or flightdeck's own. Added within v1 (#180)."} } }}, "egress": {"type": "object", "additionalProperties": {"type": "string"}, "description": "Each environment's egress policy at the start (canonical JSON); a change stops new trials."}, "trial_egress": {"type": "object", "description": "KAN-234: what the trials reach. source: closed (hangar's default for an environment that sets no egress: allowlist, no GitHub) or environment; github: whether GitHub is reachable, with a warning when it is.", "properties": {"source": {"enum": ["closed", "environment"]}, "mode": {"type": "string"}, "github": {"type": "boolean"}, "warning": {"type": "string"}}}, "pinned_at": {"type": "number"} } }, "counts": {"type": "object", "additionalProperties": {"type": "integer"}}, "trials": { "type": "array", "items": { "type": "object", "required": ["key", "task", "variant", "trial", "job", "state", "manifests", "config_digest", "verify", "grade", "stored", "drift", "links"], "properties": { "key": {"type": "string", "pattern": "^[^/]+/[^/]+/[0-9]+$", "description": "//"}, "task": {"type": "string"}, "variant": {"type": "string"}, "trial": {"type": "integer", "minimum": 1}, "job": {"type": ["string", "null"]}, "state": {"enum": ["pending", "running", "grading", "done", "cancelled", "skipped"]}, "outcome": {"type": ["string", "null"], "description": "The trial job's final state: succeeded, failed or cancelled."}, "reason": {"type": ["string", "null"]}, "session": {"type": ["string", "null"]}, "started_at": {"type": ["number", "null"]}, "finished_at": {"type": ["number", "null"]}, "turns": {"type": "integer"}, "manifests": {"type": "array", "items": {"type": ["string", "null"]}, "description": "Each agent turn's manifest digest."}, "config_digest": {"type": ["string", "null"], "description": "The first turn's manifest config_digest: equal across a variant's trials unless drift says otherwise."}, "harness_digest": {"type": ["string", "null"], "description": "The first turn's manifest harness_digest (null for trials recorded before it existed)."}, "environment_digest": {"type": ["string", "null"], "description": "The first turn's manifest environment_digest (null for trials recorded before it existed)."}, "verify": {"type": ["object", "null"], "properties": {"command": {"type": ["string", "null"]}, "exit_code": {"type": ["integer", "null"]}, "passed": {"type": ["boolean", "null"]}, "new_failures": {"type": ["array", "null"]}, "tail": {"type": ["string", "null"]}}}, "tests": {"description": "The trial's parsed test results, when hangar has them."}, "usage": {"type": ["object", "null"]}, "grade": {"type": ["object", "null"], "properties": { "command": {"type": "string"}, "exit_code": {"type": "integer"}, "tail": {"type": "string"}, "data": {}, "error": {"type": "string"}, "applied": {"type": "boolean"}, "seconds": {"type": "number"}, "files_mode": {"enum": ["pristine", "in_repo"], "description": "Where the grader's files were: outside the repo, at $HANGAR_HIDDEN_TESTS (pristine, the default), or in the repo as before (in_repo). Added within v1 (#126)."}, "network": {"enum": ["none", "mirror"], "description": "What the sealed grader's sandbox could reach: nothing, or hangar's read-only package mirror. Added within v1 (#126)."}, "trust": {"enum": ["isolated", "best_effort", "low"], "description": "How far the grade can be trusted: isolated (the uid split: hangar's user ran the command as the agent and wrote the grade itself, from the pristine copy with hangar-grade), best_effort (no uid split: the trial's code could write $HANGAR_GRADE_OUT), low (files_mode in_repo, or a command that isn't hangar-grade). Added within v1 (#126, #127)."}, "isolation": {"type": "string", "description": "split (the uid split ran the grade), else why not. Added within v1 (#127)."}, "task_invalid": {"type": "boolean", "description": "The task's hidden tests pass on its base (or its reference patch fails them): the grade can't tell a fix from no fix. Added within v1 (#127)."}, "canary_files": {"type": "integer", "description": "How many hidden test files carried the canary (files without comments, such as JSON, don't). Added within v1 (#127)."}, "runner": {"enum": ["hangar-grade", "command"], "description": "Whether the command used the safe runner (hangar-grade) or ran the tests itself. Added within v1 (#126)."}, "tamper_signals": {"type": "array", "description": "Files the trial's patch added or changed that can hook a test run before the tests (conftest.py, sitecustomize.py, usercustomize.py, *.pth, pytest settings). Signals, not refusals. Added within v1 (#126).", "items": {"type": "object", "required": ["path", "kind"], "properties": {"path": {"type": "string"}, "kind": {"enum": ["conftest", "sitecustomize", "usercustomize", "pth", "pytest-config"]}}}}, "hidden_tests": {"type": "object", "description": "The hidden tests' junit counts, when hangar-grade wrote them. Added within v1 (#126).", "properties": {"passed": {"type": "integer"}, "failed": {"type": "integer"}, "error": {"type": ["integer", "string"]}, "skipped": {"type": "integer"}, "total": {"type": "integer"}, "truncated": {"type": "boolean"}}}, "notes": {"type": "array", "items": {"type": "string"}} }}, "stored": {"type": "array", "items": {"enum": ["manifest.json", "trace.json"]}}, "drift": {"type": "array", "items": {"type": "string"}, "description": "Why this trial's configuration differs from an earlier trial of its variant; each message names the half that drifted (\"harness drift\", \"environment drift\")."}, "trust": {"enum": ["isolated", "best_effort", "low", null], "description": "The grade's trust level, at the trial's top level for filtering (null: not graded). Added within v1 (#127)."}, "task_invalid": {"type": "boolean", "description": "The task failed its base check (hidden tests pass on base, or the reference patch fails them): leave its trials out of comparisons. Added within v1 (#127)."}, "contaminated": {"type": "boolean", "description": "The grader's canary was seen in what the trial produced: the agent had the hidden tests. Added within v1 (#127)."}, "contamination": {"type": "array", "items": {"enum": ["patch", "result", "trace", "egress"]}, "description": "Where the canary was seen. Added within v1 (#127)."}, "touched_tests": {"type": "array", "items": {"type": "string"}, "description": "Existing test files the trial's work changed, renamed or deleted (the job's own check, else from its patch). Added within v1 (#127)."}, "outcome_class": {"enum": ["agent", "infra_error", "cancelled", "human", null], "description": "What the trial's outcome says about the agent: agent (its own result, counted), infra_error (infrastructure ended it: see infra_reason; excluded and retried), cancelled, human (a person stepped in). null: unknown (recorded before #166), never to be read as agent. Any turn infrastructure cut off makes the whole trial infra_error, even if its job resumed. Added within v1 (#166)."}, "infra_reason": {"enum": ["preempted", "power_down", "rate_limited", "provider_5xx", "sandbox_create", "checkpoint", "control_plane_restart", "drift", "no_login", "login_rejected", null], "description": "What infrastructure did, when outcome_class is infra_error. Added within v1 (#166). drift (#178): its turn 1 didn't run its pin set; excluded and not retried. no_login (#293): its harness had no login, so the turn wasn't started. login_rejected (KAN-178): the vendor refused its login; excluded and not retried."}, "pin_set": {"type": ["string", "null"], "pattern": "^pin-[0-9a-f]{16}$", "description": "The pin set the trial ran against (GET /api/pinsets/{id}). Added within v1 (#178)."}, "cached": {"type": "boolean", "description": "The trial is in the trial cache under its pin set and task (a counted trial of a fully pinned key). Added within v1 (#180)."}, "replay": {"type": ["string", "null"], "description": "A fork's trial: its replay (rpl-...), which replayed the run's tape to at_seq before going live (docs/model-gateway.md#replay). Added within v1 (#179)."}, "fork_of": {"type": ["string", "null"], "description": "A fork's trial: the run it forked, {session}.{run}. Added within v1 (#179)."}, "tainted": {"type": "boolean", "description": "The run it forked had an untrusted tape (#177): not promotion evidence without review. Added within v1 (#179)."}, "aa": {"type": ["object", "null"], "description": "A fork's trial with a clean prefix on the run's own harness: {production_harness, original_job, original_passed, fork_passed, reproduced}. Added within v1 (#179)."}, "excluded": {"type": "boolean", "description": "Leave this trial out of statistics: it ended in infra_error. Its retry, if any, is the next attempt under the same key. Added within v1 (#166)."}, "attempt": {"type": "integer", "minimum": 1, "description": "1 for the trial as laid out; 2 and up for its retries after infra errors (at most HANGAR_TRIAL_INFRA_RETRIES, and 10% of the experiment's trials in all). Several trials can share a key: the highest attempt is the one that counts. Added within v1 (#166)."}, "retry_of": {"type": ["string", "null"], "description": "The job id of the excluded trial this one retries; with attempt, the retry chain. Added within v1 (#166)."}, "links": {"type": "object", "properties": {"job": {"type": ["string", "null"]}, "manifest": {"type": ["string", "null"]}, "trace": {"type": ["string", "null"]}}} } } }, "notes": {"type": "array", "items": {"type": "string"}}, "task_checks": {"type": "object", "description": "Per task, the base-must-fail check, run once per experiment: the hidden tests on the pinned base without a patch (they must fail), and on the task's reference_patch when given (it must pass). Added within v1 (#127).", "additionalProperties": { "type": "object", "properties": { "state": {"enum": ["done"]}, "base_sha": {"type": ["string", "null"]}, "invalid": {"type": "boolean"}, "reason": {"type": "string"}, "base": {"type": "object"}, "reference": {"type": "object"}, "at": {"type": "number"} } }} } }