Test definition schema

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://ansight.dev/schema/test-definition.v1.schema.json",
  "title": "Ansight workspace test",
  "type": "object",
  "additionalProperties": false,
  "required": ["appId"],
  "anyOf": [{ "required": ["prompt"] }, { "required": ["taskId"] }],
  "allOf": [{
    "if": { "required": ["prompt"] },
    "then": { "not": { "required": ["taskInput"] } },
    "else": { "not": { "anyOf": [{ "required": ["validation"] }, { "required": ["hintTasks"] }] } }
  }],
  "properties": {
    "schemaVersion": {
      "const": 1,
      "description": "Test definition format version. Use 1; if omitted, version 1 is assumed."
    },
    "enabled": {
      "description": "Whether this saved test can be run. Defaults to true; set false to keep the definition without running it.",
      "type": "boolean",
      "default": true
    },
    "id": {
      "type": "string",
      "minLength": 1,
      "description": "Stable identifier for this test. If omitted, the ID is derived from its path under the tests directory."
    },
    "name": {
      "type": "string",
      "minLength": 1,
      "description": "Human-readable test name shown in test lists and results. If omitted, the filename is turned into a name."
    },
    "appId": {
      "type": "string",
      "minLength": 1,
      "description": "ID of the app this test runs against. It must match the app linked to the workspace."
    },
    "taskId": {
      "type": "string",
      "minLength": 1,
      "description": "Repository task to run. Without prompt, runs deterministically with no agent; with prompt, the agent is instructed to run the task. Target compatibility is checked before execution."
    },
    "taskInput": {
      "type": "object",
      "description": "Input arguments for a task-only test. Validated against the task input schema before launch. Only supported when prompt is omitted."
    },
    "hintTasks": {
      "type": "array",
      "description": "Optional repository task IDs suggested to the agent and preloaded ahead of search matches. Hints do not bypass starting-state, scope, or assertion checks.",
      "maxItems": 5,
      "uniqueItems": true,
      "items": { "type": "string", "minLength": 1 }
    },
    "prompt": {
      "type": "string",
      "minLength": 1,
      "description": "Actions and observable checks for the agent, in order. Omit when taskId should run directly without an agent. Supports @task/ID and @selector/ID references."
    },
    "validation": {
      "description": "Optional additional checks. Checks can also be written directly in prompt. Use a validation string, or an object with a prompt, assertions, or both.",
      "oneOf": [
        { "type": "string", "minLength": 1 },
        {
          "type": "object",
          "additionalProperties": false,
          "anyOf": [
            { "required": ["prompt"], "properties": { "prompt": { "minLength": 1 } } },
            { "required": ["assertions"], "properties": { "assertions": { "minItems": 1 } } }
          ],
          "properties": {
            "prompt": {
              "type": "string",
              "description": "Instructions for checking whether the final app state satisfies the test."
            },
            "assertions": {
              "type": "array",
              "description": "Specific outcomes the agent must verify after the scenario. Provide at least one when there is no validation prompt.",
              "items": { "type": "string", "minLength": 1 }
            }
          }
        }
      ]
    },
    "requiredSecrets": {
      "description": "Secret aliases needed by this test at run time. Each alias is resolved from the app-scoped secret store first, then from an environment variable with the same name.",
      "type": "array",
      "uniqueItems": true,
      "items": { "type": "string", "minLength": 1, "maxLength": 128 }
    }
  }
}