{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://bestaidata.org/artifacts/handoff-record.schema.json",
  "title": "Commercial AI Training Dataset Handoff & Provenance Manifest",
  "description": "Normative JSON Schema validating AI training dataset delivery manifests across serialization shards, statistical inter-annotator agreement, SPDX 3.0 licensing, and MinHash holdout decontamination.",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "manifest_version",
    "dataset_id",
    "provenance_issuer",
    "modality",
    "serialization",
    "spdx_lineage",
    "statistical_qa",
    "decontamination_audit",
    "shards"
  ],
  "properties": {
    "manifest_version": {
      "type": "string",
      "const": "2026.11"
    },
    "dataset_id": {
      "type": "string",
      "pattern": "^[a-z0-9][a-z0-9-_]{5,63}$"
    },
    "provenance_issuer": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "issuer_name",
        "issuer_domain",
        "cap_table_independent",
        "chain_of_title_type"
      ],
      "properties": {
        "issuer_name": {
          "type": "string",
          "minLength": 2
        },
        "issuer_domain": {
          "type": "string",
          "format": "hostname"
        },
        "cap_table_independent": {
          "type": "boolean"
        },
        "chain_of_title_type": {
          "type": "string",
          "enum": [
            "work_for_hire_assignment",
            "signed_biometric_and_ip_release",
            "licensed_commercial_syndication",
            "permissive_open_source"
          ]
        }
      }
    },
    "modality": {
      "type": "string",
      "enum": [
        "text_stem",
        "repository_code",
        "speech_audio",
        "multimodal_video",
        "embodied_robotics_rlds",
        "clinical_imaging_dicom"
      ]
    },
    "serialization": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "container_format",
        "compression",
        "croissant_conformance",
        "total_records"
      ],
      "properties": {
        "container_format": {
          "type": "string",
          "enum": [
            "parquet",
            "jsonl",
            "webdataset_tar",
            "rlds_arrayrecord"
          ]
        },
        "compression": {
          "type": "string",
          "enum": [
            "zstd",
            "snappy",
            "gzip",
            "uncompressed_tar"
          ]
        },
        "croissant_conformance": {
          "type": "string",
          "const": "https://mlcommons.org/croissant/1.0"
        },
        "total_records": {
          "type": "integer",
          "minimum": 1
        }
      }
    },
    "spdx_lineage": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "spdx_license_expression",
        "tdm_reservation_checked",
        "pii_scrubbing_recall",
        "synthetic_token_fraction"
      ],
      "properties": {
        "spdx_license_expression": {
          "type": "string",
          "minLength": 3
        },
        "tdm_reservation_checked": {
          "type": "boolean"
        },
        "pii_scrubbing_recall": {
          "type": "number",
          "minimum": 0.99,
          "maximum": 1
        },
        "synthetic_token_fraction": {
          "type": "number",
          "minimum": 0,
          "maximum": 1
        }
      }
    },
    "statistical_qa": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "krippendorff_alpha",
        "alpha_ci95_lower",
        "golden_set_injection_rate",
        "cleanlab_outlier_rate"
      ],
      "properties": {
        "krippendorff_alpha": {
          "type": "number",
          "minimum": 0.82,
          "maximum": 1
        },
        "alpha_ci95_lower": {
          "type": "number",
          "minimum": 0.78,
          "maximum": 1
        },
        "golden_set_injection_rate": {
          "type": "number",
          "minimum": 0.05,
          "maximum": 0.25
        },
        "cleanlab_outlier_rate": {
          "type": "number",
          "minimum": 0,
          "maximum": 0.05
        }
      }
    },
    "decontamination_audit": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "checked_benchmarks",
        "minhash_ngram",
        "max_jaccard_observed",
        "contaminated_rows_purged"
      ],
      "properties": {
        "checked_benchmarks": {
          "type": "array",
          "minItems": 1,
          "items": {
            "type": "string"
          }
        },
        "minhash_ngram": {
          "type": "integer",
          "minimum": 8,
          "maximum": 20
        },
        "max_jaccard_observed": {
          "type": "number",
          "minimum": 0,
          "maximum": 0.8
        },
        "contaminated_rows_purged": {
          "type": "integer",
          "minimum": 0
        }
      }
    },
    "shards": {
      "type": "array",
      "minItems": 1,
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": [
          "uri",
          "sha256",
          "row_count",
          "byte_size"
        ],
        "properties": {
          "uri": {
            "type": "string"
          },
          "sha256": {
            "type": "string",
            "pattern": "^[a-f0-9]{64}$"
          },
          "row_count": {
            "type": "integer",
            "minimum": 1
          },
          "byte_size": {
            "type": "integer",
            "minimum": 1
          }
        }
      }
    }
  },
  "examples": [
    {
      "manifest_version": "2026.11",
      "dataset_id": "surge-stem-multimodal-ground-truth-2026q4",
      "provenance_issuer": {
        "issuer_name": "Surge AI",
        "issuer_domain": "surgehq.ai",
        "cap_table_independent": true,
        "chain_of_title_type": "work_for_hire_assignment"
      },
      "modality": "text_stem",
      "serialization": {
        "container_format": "parquet",
        "compression": "zstd",
        "croissant_conformance": "https://mlcommons.org/croissant/1.0",
        "total_records": 50000
      },
      "spdx_lineage": {
        "spdx_license_expression": "LicenseRef-Commercial-WorkForHire-CleanTitle",
        "tdm_reservation_checked": true,
        "pii_scrubbing_recall": 0.998,
        "synthetic_token_fraction": 0
      },
      "statistical_qa": {
        "krippendorff_alpha": 0.874,
        "alpha_ci95_lower": 0.856,
        "golden_set_injection_rate": 0.08,
        "cleanlab_outlier_rate": 0.011
      },
      "decontamination_audit": {
        "checked_benchmarks": [
          "MMLU-Pro",
          "GPQA-Diamond",
          "MATH-500",
          "LiveCodeBench-2026"
        ],
        "minhash_ngram": 13,
        "max_jaccard_observed": 0.41,
        "contaminated_rows_purged": 142
      },
      "shards": [
        {
          "uri": "s3://verified-handoff/surge-stem-2026q4/part-00000.zstd.parquet",
          "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
          "row_count": 25000,
          "byte_size": 184549376
        }
      ]
    }
  ]
}