{
  "benchmark_version": 1,
  "manifest_sha256": "2610a148aaa31dd01cb842acdd5233405e4925025f29a98e13fd3b7dfa364fd0",
  "source_commit": "03c4fb38b51364caf42cde2580a3e9bb933a505c",
  "inference_source_sha256": {
    "src/services/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "src/services/approvals.py": "8b8ec9ca569ffcd56ca8b13e03584d33c246774c654020cdcc27af6e1fa92dbe",
    "src/services/certification.py": "8c6cf45bb6c0a5fbed5616095064a61ef8506abbd57c51aa4b042fb0342ce07a",
    "src/services/connection.py": "d2d29e70e25bb2b6c8f9583ea39f83ff6f9629b5c17074a84b1e90cf6ee2f6c5",
    "src/services/entities.py": "1f1f71ffd0eee6d2e1a2739ff12c83d26a3796251deb49f78525c148649aa117",
    "src/services/gates.py": "256b7d3b4ed201595b201bfc79fdaf25069ae3873cfc5833be23c7f133b83406",
    "src/services/generation.py": "ae2212675714018b321b75c8e87e7fe88b07b87f04ce22bea39f7581a2e26c3d",
    "src/services/joins.py": "2c91aff9547067b8a3da017512bff6be02c6c53e4a818fddd479d5faa3135571",
    "src/services/judge.py": "36a89d41a51883d9e19eed49018189bdc80a338a5d2640719f8cc45f7bf24c69",
    "src/services/mapping.py": "6f91f862d942164430f9a613b9a43518628f5b3b08dbfcee5183f7e96dde6a6c",
    "src/services/pii.py": "6b5f8d23b2e85a7ccade54c075611bb5451e6fd678ba4fd566280678d18e543c",
    "src/services/profiling.py": "465855d2eec0b435b35705152f76b65a7fedef8bf354635ff49a696ab67c444b",
    "src/services/publish.py": "f10fc68674bb3a6cf8b2ad931828aba6ae71f9083341eeaa6a3128ef58bed6c1",
    "src/services/sandbox.py": "1f109494643799083b4f19d239a367fcc2fd1aa7975ef7f29427be5fbcf3b6f2",
    "src/services/semantic.py": "06dd93bcb89d83fb5a151fac33aca4bb47c264c7e017cde746707c8a3af8b6e7",
    "src/services/text.py": "9cf8bb14ac4279176b6f3c86d6a0d2f04ac631262eb8c5edbfcbd21ff34efa9e"
  },
  "provenance": "Separately agent-authored frozen synthetic cases; not external human or fully blinded evaluation",
  "confidence_interpretation": "Empirical categorical correctness; heuristic scores are not probabilities",
  "label_separation": "Labels loaded only by evaluator after label-guarded inference subprocess exits; not an OS sandbox",
  "accuracy_floor": null,
  "cases": {
    "structural": {
      "entity_correct": 2,
      "entity_total": 2,
      "entity_accuracy": 1.0,
      "mapping_correct": 9,
      "mapping_proposed": 10,
      "mapping_labelled_positive": 9,
      "mapping_precision": 0.9,
      "mapping_recall": 1.0,
      "mapping_coverage": 1.0,
      "abstained_fields": [],
      "ambiguous_fields": 1,
      "correct_ambiguous_abstentions": 0,
      "review_proposals": 10,
      "review_rate": 1.0,
      "unresolved_field_burden": 0,
      "join_correct": 1,
      "join_proposed": 1,
      "join_expected": 1,
      "join_precision": 1.0,
      "join_recall": 1.0,
      "units_correct": 1,
      "units_attempted": 1,
      "units_expected": 1,
      "unit_accuracy_when_proposed": 1.0,
      "confidence_reliability": {
        "low": {
          "count": 4,
          "correct": 3,
          "accuracy": 0.75
        },
        "medium": {
          "count": 6,
          "correct": 6,
          "accuracy": 1.0
        }
      },
      "mapping_errors": [
        {
          "source": "raw.r17.batch_marker",
          "predicted": "invoice.status",
          "expected": null
        }
      ],
      "unit_errors": [],
      "join_errors": [],
      "integrity_passed": true,
      "unsafe_decisions": [],
      "safety_passed": true
    },
    "opaque": {
      "entity_correct": 2,
      "entity_total": 2,
      "entity_accuracy": 1.0,
      "mapping_correct": 0,
      "mapping_proposed": 0,
      "mapping_labelled_positive": 0,
      "mapping_precision": null,
      "mapping_recall": null,
      "mapping_coverage": 0.0,
      "abstained_fields": [
        "raw.z8.d3",
        "raw.z8.n7",
        "raw.z8.x1",
        "raw.z8.x2",
        "raw.z9.d3",
        "raw.z9.n7",
        "raw.z9.x1",
        "raw.z9.x2"
      ],
      "ambiguous_fields": 8,
      "correct_ambiguous_abstentions": 8,
      "review_proposals": 0,
      "review_rate": null,
      "unresolved_field_burden": 8,
      "join_correct": 0,
      "join_proposed": 0,
      "join_expected": 0,
      "join_precision": null,
      "join_recall": null,
      "units_correct": 0,
      "units_attempted": 0,
      "units_expected": 0,
      "unit_accuracy_when_proposed": null,
      "confidence_reliability": {},
      "mapping_errors": [],
      "unit_errors": [],
      "join_errors": [],
      "integrity_passed": true,
      "unsafe_decisions": [],
      "safety_passed": true
    },
    "noisy": {
      "entity_correct": 3,
      "entity_total": 3,
      "entity_accuracy": 1.0,
      "mapping_correct": 9,
      "mapping_proposed": 9,
      "mapping_labelled_positive": 9,
      "mapping_precision": 1.0,
      "mapping_recall": 1.0,
      "mapping_coverage": 1.0,
      "abstained_fields": [],
      "ambiguous_fields": 0,
      "correct_ambiguous_abstentions": 0,
      "review_proposals": 3,
      "review_rate": 0.333333,
      "unresolved_field_burden": 0,
      "join_correct": 1,
      "join_proposed": 1,
      "join_expected": 1,
      "join_precision": 1.0,
      "join_recall": 1.0,
      "units_correct": 1,
      "units_attempted": 1,
      "units_expected": 1,
      "unit_accuracy_when_proposed": 1.0,
      "confidence_reliability": {
        "high": {
          "count": 7,
          "correct": 7,
          "accuracy": 1.0
        },
        "medium": {
          "count": 2,
          "correct": 2,
          "accuracy": 1.0
        }
      },
      "mapping_errors": [],
      "unit_errors": [],
      "join_errors": [],
      "integrity_passed": true,
      "unsafe_decisions": [],
      "safety_passed": true
    },
    "units": {
      "entity_correct": 3,
      "entity_total": 3,
      "entity_accuracy": 1.0,
      "mapping_correct": 14,
      "mapping_proposed": 14,
      "mapping_labelled_positive": 14,
      "mapping_precision": 1.0,
      "mapping_recall": 1.0,
      "mapping_coverage": 1.0,
      "abstained_fields": [],
      "ambiguous_fields": 0,
      "correct_ambiguous_abstentions": 0,
      "review_proposals": 4,
      "review_rate": 0.285714,
      "unresolved_field_burden": 0,
      "join_correct": 1,
      "join_proposed": 1,
      "join_expected": 1,
      "join_precision": 1.0,
      "join_recall": 1.0,
      "units_correct": 4,
      "units_attempted": 4,
      "units_expected": 4,
      "unit_accuracy_when_proposed": 1.0,
      "confidence_reliability": {
        "high": {
          "count": 12,
          "correct": 12,
          "accuracy": 1.0
        },
        "low": {
          "count": 1,
          "correct": 1,
          "accuracy": 1.0
        },
        "medium": {
          "count": 1,
          "correct": 1,
          "accuracy": 1.0
        }
      },
      "mapping_errors": [],
      "unit_errors": [],
      "join_errors": [],
      "integrity_passed": true,
      "unsafe_decisions": [],
      "safety_passed": true
    }
  },
  "safety_passed": true
}