{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://course.dragonai.tech/datasets/ai-product-evaluation-suite-schema-v1.json",
  "datasetId": "ai-product-evaluation-suite-schema-v1",
  "schemaVersion": "1.0.0",
  "title": "AI 产品评测套件 JSON Schema v1",
  "description": "用于自动校验产品级 AI 评测套件必需字段、枚举值、指标定义、切片规则、测试样例与发布决策结构的 JSON Schema Draft 2020-12 文件。",
  "type": "object",
  "required": [
    "suiteId",
    "suiteVersion",
    "productScenario",
    "deploymentContext",
    "intendedUsers",
    "excludedUses",
    "riskTolerance",
    "severityDefinitions",
    "systemUnderTest",
    "evaluationDataset",
    "dataCutoff",
    "owner",
    "reviewers",
    "evaluatedAt",
    "metricDefinitions",
    "sliceDefinitions",
    "suiteGateRules",
    "cases",
    "releaseDecision"
  ],
  "properties": {
    "suiteId": {
      "type": "string"
    },
    "suiteVersion": {
      "type": "string"
    },
    "productScenario": {
      "type": "string"
    },
    "deploymentContext": {
      "type": "string"
    },
    "intendedUsers": {
      "type": "array",
      "items": {
        "type": "string"
      }
    },
    "excludedUses": {
      "type": "array",
      "items": {
        "type": "string"
      }
    },
    "riskTolerance": {
      "type": "string"
    },
    "severityDefinitions": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/severityDefinition"
      }
    },
    "systemUnderTest": {
      "$ref": "#/$defs/systemUnderTest"
    },
    "evaluationDataset": {
      "type": "object",
      "required": [
        "datasetVersion",
        "datasetHash"
      ],
      "properties": {
        "datasetVersion": {
          "type": "string"
        },
        "datasetHash": {
          "type": "string"
        }
      },
      "additionalProperties": false
    },
    "dataCutoff": {
      "type": "string"
    },
    "owner": {
      "type": "string"
    },
    "reviewers": {
      "type": "array",
      "items": {
        "type": "string"
      }
    },
    "evaluatedAt": {
      "type": "string"
    },
    "metricDefinitions": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/metricDefinition"
      }
    },
    "sliceDefinitions": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/sliceDefinition"
      }
    },
    "suiteGateRules": {
      "$ref": "#/$defs/suiteGateRules"
    },
    "cases": {
      "type": "array",
      "items": {
        "$ref": "#/$defs/evaluationCase"
      }
    },
    "releaseDecision": {
      "$ref": "#/$defs/releaseDecision"
    }
  },
  "$defs": {
    "severity": {
      "enum": [
        "low",
        "medium",
        "high",
        "critical"
      ]
    },
    "severityDefinition": {
      "type": "object",
      "required": [
        "severity",
        "blocksReleaseByDefault",
        "humanOverrideAllowed",
        "requiredOverrideRoles"
      ],
      "properties": {
        "severity": {
          "$ref": "#/$defs/severity"
        },
        "blocksReleaseByDefault": {
          "type": "boolean"
        },
        "humanOverrideAllowed": {
          "type": "boolean"
        },
        "requiredOverrideRoles": {
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      },
      "additionalProperties": false
    },
    "systemUnderTest": {
      "type": "object",
      "required": [
        "modelId",
        "modelVersion",
        "promptVersion",
        "applicationVersionOrCommit",
        "retrievalCorpusVersion",
        "embeddingVersion",
        "vectorIndexVersion",
        "toolVersions",
        "policyVersion",
        "configurationHash"
      ],
      "properties": {
        "modelId": {
          "type": "string"
        },
        "modelVersion": {
          "type": "string"
        },
        "promptVersion": {
          "type": "string"
        },
        "applicationVersionOrCommit": {
          "type": "string"
        },
        "retrievalCorpusVersion": {
          "type": "string"
        },
        "embeddingVersion": {
          "type": "string"
        },
        "vectorIndexVersion": {
          "type": "string"
        },
        "toolVersions": {
          "type": "object",
          "additionalProperties": {
            "type": "string"
          }
        },
        "policyVersion": {
          "type": "string"
        },
        "configurationHash": {
          "type": "string"
        }
      },
      "additionalProperties": false
    },
    "metricDefinition": {
      "type": "object",
      "required": [
        "metricId",
        "name",
        "measurementMethod",
        "direction",
        "comparator",
        "passThreshold",
        "unit",
        "aggregation",
        "evaluationScope",
        "applicableSlices",
        "minimumSampleSize",
        "uncertaintyMethod",
        "missingResultPolicy"
      ],
      "properties": {
        "metricId": {
          "type": "string"
        },
        "name": {
          "type": "string"
        },
        "measurementMethod": {
          "type": "string"
        },
        "direction": {
          "enum": [
            "higher_is_better",
            "lower_is_better"
          ]
        },
        "comparator": {
          "enum": [
            "gte",
            "gt",
            "lte",
            "lt",
            "eq"
          ]
        },
        "passThreshold": {
          "anyOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ]
        },
        "unit": {
          "type": "string"
        },
        "aggregation": {
          "type": "string"
        },
        "evaluationScope": {
          "enum": [
            "case",
            "slice",
            "suite",
            "case_and_slice",
            "slice_and_suite",
            "case_slice_and_suite"
          ]
        },
        "applicableSlices": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "minimumSampleSize": {
          "type": "integer",
          "minimum": 1
        },
        "uncertaintyMethod": {
          "type": "string"
        },
        "missingResultPolicy": {
          "type": "string"
        }
      },
      "additionalProperties": false
    },
    "sliceDefinition": {
      "type": "object",
      "required": [
        "sliceId",
        "description",
        "caseSelectionRule",
        "minimumCaseCount",
        "requiredMetricIds",
        "missingSlicePolicy"
      ],
      "properties": {
        "sliceId": {
          "type": "string"
        },
        "description": {
          "type": "string"
        },
        "caseSelectionRule": {
          "type": "string"
        },
        "minimumCaseCount": {
          "type": "integer",
          "minimum": 1
        },
        "requiredMetricIds": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "missingSlicePolicy": {
          "type": "string"
        }
      },
      "additionalProperties": false
    },
    "suiteGateRules": {
      "type": "object",
      "required": [
        "ruleVersion",
        "casePassRule",
        "missingMetricPolicy",
        "missingRequiredSlicePolicy",
        "blockingRules",
        "sliceGateRules"
      ],
      "properties": {
        "ruleVersion": {
          "type": "string"
        },
        "casePassRule": {
          "type": "string"
        },
        "missingMetricPolicy": {
          "type": "string"
        },
        "missingRequiredSlicePolicy": {
          "type": "string"
        },
        "blockingRules": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/blockingRule"
          }
        },
        "sliceGateRules": {
          "type": "array",
          "items": {
            "$ref": "#/$defs/sliceGateRule"
          }
        }
      },
      "additionalProperties": false
    },
    "blockingRule": {
      "type": "object",
      "required": [
        "ruleId",
        "condition",
        "action",
        "humanOverrideAllowed",
        "requiredApproverRoles"
      ],
      "properties": {
        "ruleId": {
          "type": "string"
        },
        "condition": {
          "type": "string"
        },
        "action": {
          "enum": [
            "block_release",
            "require_review",
            "warn"
          ]
        },
        "humanOverrideAllowed": {
          "type": "boolean"
        },
        "requiredApproverRoles": {
          "type": "array",
          "items": {
            "type": "string"
          }
        }
      },
      "additionalProperties": false
    },
    "sliceGateRule": {
      "type": "object",
      "required": [
        "sliceId",
        "metricId",
        "aggregation",
        "comparator",
        "threshold",
        "minimumSampleSize",
        "failureAction"
      ],
      "properties": {
        "sliceId": {
          "type": "string"
        },
        "metricId": {
          "type": "string"
        },
        "aggregation": {
          "type": "string"
        },
        "comparator": {
          "enum": [
            "gte",
            "gt",
            "lte",
            "lt",
            "eq"
          ]
        },
        "threshold": {
          "anyOf": [
            {
              "type": "number"
            },
            {
              "type": "null"
            }
          ]
        },
        "minimumSampleSize": {
          "type": "integer",
          "minimum": 1
        },
        "failureAction": {
          "enum": [
            "block_release",
            "require_review",
            "warn"
          ]
        }
      },
      "additionalProperties": false
    },
    "humanReview": {
      "type": "object",
      "required": [
        "required",
        "reviewerId",
        "reviewerRole",
        "rubricVersion",
        "reviewedAt",
        "decision",
        "decisionReason",
        "secondReviewerId",
        "adjudication"
      ],
      "properties": {
        "required": {
          "type": "boolean"
        },
        "reviewerId": {
          "type": "string"
        },
        "reviewerRole": {
          "type": "string"
        },
        "rubricVersion": {
          "type": "string"
        },
        "reviewedAt": {
          "type": "string"
        },
        "decision": {
          "enum": [
            "not_reviewed",
            "pass",
            "fail",
            "needs_adjudication"
          ]
        },
        "decisionReason": {
          "type": "string"
        },
        "secondReviewerId": {
          "type": "string"
        },
        "adjudication": {
          "type": "string"
        }
      },
      "additionalProperties": false
    },
    "overrideApproval": {
      "type": "object",
      "required": [
        "approverId",
        "approverRole",
        "reason",
        "approvedAt"
      ],
      "properties": {
        "approverId": {
          "type": "string"
        },
        "approverRole": {
          "type": "string"
        },
        "reason": {
          "type": "string"
        },
        "approvedAt": {
          "type": "string"
        }
      },
      "additionalProperties": false
    },
    "caseDecision": {
      "type": "object",
      "required": [
        "ruleVersion",
        "computedPass",
        "blockingReasons",
        "overridden",
        "overrideApproval"
      ],
      "properties": {
        "ruleVersion": {
          "type": "string"
        },
        "computedPass": {
          "anyOf": [
            {
              "type": "boolean"
            },
            {
              "type": "null"
            }
          ]
        },
        "blockingReasons": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "overridden": {
          "type": "boolean"
        },
        "overrideApproval": {
          "$ref": "#/$defs/overrideApproval"
        }
      },
      "additionalProperties": false
    },
    "evaluationCase": {
      "type": "object",
      "required": [
        "caseId",
        "userTask",
        "input",
        "referenceOutput",
        "context",
        "expectedBehaviors",
        "prohibitedBehaviors",
        "severity",
        "sliceTags",
        "dataProvenance",
        "consentOrLicense",
        "metricIds",
        "actualOutput",
        "metricResults",
        "humanReview",
        "caseDecision",
        "failureCategory",
        "evidenceUrl"
      ],
      "properties": {
        "caseId": {
          "type": "string"
        },
        "userTask": {
          "type": "string"
        },
        "input": {
          "type": "string"
        },
        "referenceOutput": {
          "type": "string"
        },
        "context": {
          "type": "string"
        },
        "expectedBehaviors": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "prohibitedBehaviors": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "severity": {
          "$ref": "#/$defs/severity"
        },
        "sliceTags": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "dataProvenance": {
          "type": "string"
        },
        "consentOrLicense": {
          "type": "string"
        },
        "metricIds": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "actualOutput": {
          "type": "string"
        },
        "metricResults": {
          "type": "object"
        },
        "humanReview": {
          "$ref": "#/$defs/humanReview"
        },
        "caseDecision": {
          "$ref": "#/$defs/caseDecision"
        },
        "failureCategory": {
          "type": "string"
        },
        "evidenceUrl": {
          "type": "string"
        }
      },
      "additionalProperties": false
    },
    "releaseDecision": {
      "type": "object",
      "required": [
        "decision",
        "evaluatedArtifactHash",
        "blockingFailures",
        "knownLimitations",
        "rolloutScope",
        "monitoringMetrics",
        "triggerThresholds",
        "observationWindow",
        "rollbackTargetVersion",
        "rollbackProcedure",
        "rollbackOwner",
        "postRollbackVerification",
        "approvedById",
        "approvedByRole",
        "approvedAt",
        "reviewAfter",
        "approvalExpiresAt"
      ],
      "properties": {
        "decision": {
          "enum": [
            "not_decided",
            "approved",
            "approved_with_limits",
            "blocked"
          ]
        },
        "evaluatedArtifactHash": {
          "type": "string"
        },
        "blockingFailures": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "knownLimitations": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "rolloutScope": {
          "type": "string"
        },
        "monitoringMetrics": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "triggerThresholds": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "observationWindow": {
          "type": "string"
        },
        "rollbackTargetVersion": {
          "type": "string"
        },
        "rollbackProcedure": {
          "type": "string"
        },
        "rollbackOwner": {
          "type": "string"
        },
        "postRollbackVerification": {
          "type": "string"
        },
        "approvedById": {
          "type": "string"
        },
        "approvedByRole": {
          "type": "string"
        },
        "approvedAt": {
          "type": "string"
        },
        "reviewAfter": {
          "type": "string"
        },
        "approvalExpiresAt": {
          "type": "string"
        }
      },
      "additionalProperties": false
    }
  },
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "licenseScope": "This JSON Schema authored by DragonAI. Third-party source material is excluded.",
  "url": "https://course.dragonai.tech/datasets/ai-product-evaluation-suite-schema-v1.json",
  "creator": "烛龙智元内容研究组"
}