schema_version: slac.sota_audit_protocol/v0.1
protocol_id: kitti-camera-lidar-check-v2
scope:
  modalities:
  - camera
  - lidar
  quantities:
  - rotation
  category: targetless_camera_lidar_check
claim_text: 'On 18 KITTI raw drives of 2011_09_28, 2011_09_29, 2011_09_30 and 2011_10_03
  that no Calibrex camera-LiDAR method had read, calibrex check --pairs camera-lidar
  on each drive''s first 600 frames gives honest per-axis verdicts against that day''s
  KITTI calibration (calib_velo_to_cam with the rectification of calib_cam_to_cam):
  it never flags that calibration (no warn or fail); every rotation axis it judges
  there (reported std at most 0.5 deg and its 2 deg known-bad control detected on
  held-out frames at t >= 2.5) is within 0.75 deg of it, and it judges at least 50
  percent of the axes; started at that calibration with the camera turned 1 or 3 deg
  about one of its axes, every axis it judges is within 1.5 deg of the day''s calibration;
  it flags a 3 deg turn about an axis it judged on the drive in at least 90 percent
  of the trials and never judges such a turned axis right; and it flags a 1 deg turn
  in at least 75 percent of the trials.'
minimum_dataset_families: 1
minimum_independent_rigs: 1
requirements:
- requirement_id: no-failed-drives
  phase: audit
  description: no drive fails
  dataset_family: kitti_raw
  independent_rig: true
  rig_id: kitti-raw-car
  evidence_path: kitti_camera_lidar_v2_drives_benchmark.yaml
  evidence_sha256: 6fcc39b6250c46e97b906c52c33e9ad373f0a4a498953a95075547e1ebb85120
  method_id: calibrex_check
  statistic: failure_rate
  comparison: less_equal
  threshold: 0.0
  required: true
- requirement_id: vendor-calibration-not-flagged
  phase: audit
  description: the check flags the day's KITTI calibration on no scored drive
  dataset_family: kitti_raw
  independent_rig: false
  evidence_path: kitti_camera_lidar_v2_baseline_benchmark.yaml
  evidence_sha256: 451309b6fcebbb35ae15aebb33b55bd28e7aa6c3f97364a11b3463df239e41a0
  method_id: calibrex_check
  metric: vendor_flagged
  statistic: metric_maximum
  comparison: less_equal
  threshold: 0.0
  required: true
- requirement_id: judged-axis-accuracy
  phase: audit
  description: every judged baseline axis is within 0.75 deg of KITTI's calibration
  dataset_family: kitti_raw
  independent_rig: false
  evidence_path: kitti_camera_lidar_v2_axes_benchmark.yaml
  evidence_sha256: fe948dca79065e23bcbc86d346d214e84bafa3cf968c48b418113f04ed8ffe35
  method_id: calibrex_check
  metric: axis_error_deg_abs
  statistic: metric_maximum
  comparison: less_equal
  threshold: 0.75
  required: true
- requirement_id: judged-axis-coverage
  phase: audit
  description: on average at most 1.5 of a scored drive's 3 baseline axes are not
    judged
  dataset_family: kitti_raw
  independent_rig: false
  evidence_path: kitti_camera_lidar_v2_coverage_benchmark.yaml
  evidence_sha256: 795d7e5b468e8bc30ade366e0842b20f4565dc248100bcdffeefae7448947a36
  method_id: calibrex_check
  metric: unjudged_axes
  statistic: metric_mean
  comparison: less_equal
  threshold: 1.5
  required: true
- requirement_id: turned-judged-axis-accuracy
  phase: audit
  description: every axis judged in a turned run is within 1.5 deg of KITTI's calibration
  dataset_family: kitti_raw
  independent_rig: false
  evidence_path: kitti_camera_lidar_v2_turned_benchmark.yaml
  evidence_sha256: 3aa0486334209efd5b0bfc955cfc10c1e50aeb3b3ffcb0f79bd3ff1e09afec09
  method_id: calibrex_check
  metric: axis_error_deg_abs
  statistic: metric_maximum
  comparison: less_equal
  threshold: 1.5
  required: true
- requirement_id: turn-3deg-flagged
  phase: audit
  description: at most 0.1 of the 3 deg turns about judged axes are not flagged
  dataset_family: kitti_raw
  independent_rig: false
  evidence_path: kitti_camera_lidar_v2_turn3_benchmark.yaml
  evidence_sha256: 1e9b30ad17e496b13c7c206f9e038ada81062386ccbb0f39670e6584b7656301
  method_id: calibrex_check
  statistic: failure_rate
  comparison: less_equal
  threshold: 0.1
  required: true
- requirement_id: turn-3deg-never-judged-right
  phase: audit
  description: the check never judges an axis turned 3 deg to be right
  dataset_family: kitti_raw
  independent_rig: false
  evidence_path: kitti_camera_lidar_v2_turn3_judged_right_benchmark.yaml
  evidence_sha256: dc8623b25dc1bddd723fb53ff4d615dc081b2096961c0ac81deddf4b89b36d0f
  method_id: calibrex_check
  metric: turn_judged_right
  statistic: metric_maximum
  comparison: less_equal
  threshold: 0.0
  required: true
- requirement_id: turn-1deg-flagged
  phase: audit
  description: at most 0.25 of the 1 deg turns about judged axes are not flagged
  dataset_family: kitti_raw
  independent_rig: false
  evidence_path: kitti_camera_lidar_v2_turn1_benchmark.yaml
  evidence_sha256: 6b22bb0d75f615748285c29c218e0e753b7c812ad9ab6655856888b018b9ce1d
  method_id: calibrex_check
  statistic: failure_rate
  comparison: less_equal
  threshold: 0.25
  required: true
- requirement_id: scored-drives-coverage
  phase: audit
  description: at least 12 of the evaluation drives are scored
  dataset_family: kitti_raw
  independent_rig: false
  evidence_path: kitti_camera_lidar_v2_evaluation_benchmark.yaml
  evidence_sha256: 50bb6a145f3bde178980c6193adc2bd9c090c2668321b23b7f802a628b2fe986
  method_id: calibrex_check
  statistic: failure_rate
  comparison: less_equal
  threshold: 0.3333333333333333
  required: true
provenance:
  generator: tools/build_kitti_camera_lidar_v2_audit.py
  generator_version: 0.5.1
  command:
  - python
  - tools/build_kitti_camera_lidar_v2_audit.py
  - SCORES.json
  source_sha256:
    preregistration: f5399653825623ea42a5890cac9dc4fe6390d5ffe9241e2b977cb9140abd3afa
    scores: 18c545dbb943b5a312c3646243d59ca16227210b1dfd85f167c56a8f9a006baf
  created_at: '2026-10-15T01:40:23.106699+00:00'