spatialrust_io/
manifest.rs

1//! Checksummed file receipts and dataset manifests.
2
3use std::fs::File;
4use std::io::{BufReader, Read};
5use std::path::{Path, PathBuf};
6
7use serde::{Deserialize, Serialize};
8use sha2::{Digest, Sha256};
9
10use crate::IoError;
11
12/// Current JSON schema version for [`DatasetManifest`].
13pub const DATASET_MANIFEST_VERSION: u32 = 1;
14
15/// Logical role of a file in a dataset operation.
16#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
17#[serde(rename_all = "snake_case")]
18pub enum ReceiptRole {
19    /// File consumed by an operation.
20    Input,
21    /// File produced by an operation.
22    Output,
23    /// File associated with an operation but not consumed or produced by it.
24    Auxiliary,
25}
26
27/// Size and SHA-256 receipt for one local file or URI source.
28#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
29pub struct FileReceipt {
30    /// Role of the source in the operation.
31    pub role: ReceiptRole,
32    /// Resolved local path or URI written as a path-like JSON string.
33    pub path: PathBuf,
34    /// Number of bytes observed while hashing a local file.
35    #[serde(skip_serializing_if = "Option::is_none")]
36    pub size_bytes: Option<u64>,
37    /// Lowercase hexadecimal SHA-256 digest for a local file.
38    #[serde(skip_serializing_if = "Option::is_none")]
39    pub sha256: Option<String>,
40}
41
42impl FileReceipt {
43    /// Hashes a local file and returns its size/checksum receipt.
44    pub fn from_path(role: ReceiptRole, path: impl AsRef<Path>) -> Result<Self, IoError> {
45        let path = path.as_ref();
46        let file = File::open(path)?;
47        let mut reader = BufReader::new(file);
48        let mut hasher = Sha256::new();
49        let mut bytes = 0_u64;
50        let mut buffer = [0_u8; 64 * 1024];
51
52        loop {
53            let read = reader.read(&mut buffer)?;
54            if read == 0 {
55                break;
56            }
57            hasher.update(&buffer[..read]);
58            bytes = bytes.checked_add(read as u64).ok_or_else(|| {
59                IoError::Manifest(format!("file size overflow while hashing `{}`", path.display()))
60            })?;
61        }
62
63        let digest = hasher.finalize();
64        Ok(Self {
65            role,
66            path: path.to_path_buf(),
67            size_bytes: Some(bytes),
68            sha256: Some(hex_digest(&digest)),
69        })
70    }
71
72    /// Records a URI source whose bytes were not materialized locally.
73    #[must_use]
74    pub fn from_uri(role: ReceiptRole, uri: impl Into<PathBuf>) -> Self {
75        Self { role, path: uri.into(), size_bytes: None, sha256: None }
76    }
77}
78
79/// JSON manifest containing the files associated with one dataset operation.
80#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
81pub struct DatasetManifest {
82    /// Version of the manifest JSON schema.
83    pub version: u32,
84    /// File and URI receipts in operation order.
85    pub entries: Vec<FileReceipt>,
86}
87
88/// Summary returned after re-hashing every local manifest entry.
89#[derive(Clone, Copy, Debug, Eq, PartialEq)]
90pub struct ManifestValidation {
91    /// Number of local files re-hashed successfully.
92    pub checked_local_files: u64,
93    /// Number of URI entries that intentionally have no local checksum.
94    pub uri_entries: u64,
95    /// Total bytes observed across checked local files.
96    pub total_bytes: u64,
97}
98
99impl Default for DatasetManifest {
100    fn default() -> Self {
101        Self::new()
102    }
103}
104
105impl DatasetManifest {
106    /// Creates an empty manifest at the current schema version.
107    #[must_use]
108    pub const fn new() -> Self {
109        Self { version: DATASET_MANIFEST_VERSION, entries: Vec::new() }
110    }
111
112    /// Adds a checksummed local file receipt.
113    pub fn add_file(&mut self, role: ReceiptRole, path: impl AsRef<Path>) -> Result<(), IoError> {
114        self.entries.push(FileReceipt::from_path(role, path)?);
115        Ok(())
116    }
117
118    /// Adds a URI receipt without claiming a local byte count or checksum.
119    pub fn add_uri(&mut self, role: ReceiptRole, uri: impl Into<PathBuf>) {
120        self.entries.push(FileReceipt::from_uri(role, uri));
121    }
122
123    /// Reads a JSON manifest from disk without trusting its file receipts.
124    pub fn read_json(path: impl AsRef<Path>) -> Result<Self, IoError> {
125        let path = path.as_ref();
126        let text = std::fs::read_to_string(path).map_err(|error| {
127            IoError::Manifest(format!("cannot read dataset manifest `{}`: {error}", path.display()))
128        })?;
129        serde_json::from_str(&text).map_err(|error| {
130            IoError::Manifest(format!(
131                "cannot parse dataset manifest `{}`: {error}",
132                path.display()
133            ))
134        })
135    }
136
137    /// Re-hashes local entries and rejects missing, changed, or partial receipts.
138    pub fn validate_local_files(&self) -> Result<ManifestValidation, IoError> {
139        if self.version != DATASET_MANIFEST_VERSION {
140            return Err(IoError::Manifest(format!(
141                "unsupported dataset manifest version {}; expected {}",
142                self.version, DATASET_MANIFEST_VERSION
143            )));
144        }
145        let mut checked_local_files = 0_u64;
146        let mut uri_entries = 0_u64;
147        let mut total_bytes = 0_u64;
148        for entry in &self.entries {
149            match (&entry.size_bytes, &entry.sha256) {
150                (Some(expected_size), Some(expected_sha256)) => {
151                    let actual = FileReceipt::from_path(entry.role, &entry.path)?;
152                    if actual.size_bytes != entry.size_bytes {
153                        return Err(IoError::Manifest(format!(
154                            "size mismatch for `{}`: expected {:?}, observed {:?}",
155                            entry.path.display(),
156                            entry.size_bytes,
157                            actual.size_bytes
158                        )));
159                    }
160                    if actual.sha256.as_deref() != Some(expected_sha256.as_str()) {
161                        return Err(IoError::Manifest(format!(
162                            "checksum mismatch for `{}`: expected {}, observed {}",
163                            entry.path.display(),
164                            expected_sha256,
165                            actual.sha256.as_deref().unwrap_or("<missing>")
166                        )));
167                    }
168                    checked_local_files = checked_local_files.checked_add(1).ok_or_else(|| {
169                        IoError::Manifest(
170                            "local file count overflow while validating manifest".into(),
171                        )
172                    })?;
173                    total_bytes = total_bytes.checked_add(*expected_size).ok_or_else(|| {
174                        IoError::Manifest(
175                            "total byte count overflow while validating manifest".into(),
176                        )
177                    })?;
178                }
179                (None, None) if is_uri(&entry.path) => {
180                    uri_entries = uri_entries.checked_add(1).ok_or_else(|| {
181                        IoError::Manifest("URI count overflow while validating manifest".into())
182                    })?;
183                }
184                (None, None) => {
185                    return Err(IoError::Manifest(format!(
186                        "manifest entry `{}` has no checksum and is not a URI",
187                        entry.path.display()
188                    )));
189                }
190                _ => {
191                    return Err(IoError::Manifest(format!(
192                        "manifest entry `{}` has a partial size/checksum receipt",
193                        entry.path.display()
194                    )));
195                }
196            }
197        }
198        Ok(ManifestValidation { checked_local_files, uri_entries, total_bytes })
199    }
200
201    /// Serializes this manifest as pretty-printed JSON.
202    pub fn to_json(&self) -> Result<String, IoError> {
203        serde_json::to_string_pretty(self).map_err(|error| {
204            IoError::Manifest(format!("cannot serialize dataset manifest: {error}"))
205        })
206    }
207
208    /// Writes this manifest as JSON, creating its parent directory if needed.
209    pub fn write_json(&self, path: impl AsRef<Path>) -> Result<(), IoError> {
210        let path = path.as_ref();
211        if let Some(parent) = path.parent() {
212            if !parent.as_os_str().is_empty() {
213                std::fs::create_dir_all(parent)?;
214            }
215        }
216        std::fs::write(path, format!("{}\n", self.to_json()?))?;
217        Ok(())
218    }
219}
220
221fn hex_digest(bytes: &[u8]) -> String {
222    let mut output = String::with_capacity(bytes.len() * 2);
223    for byte in bytes {
224        output.push_str(&format!("{byte:02x}"));
225    }
226    output
227}
228
229fn is_uri(path: &Path) -> bool {
230    path.to_string_lossy().contains("://")
231}
232
233#[cfg(test)]
234mod tests {
235    use super::{DatasetManifest, FileReceipt, ReceiptRole, DATASET_MANIFEST_VERSION};
236
237    #[test]
238    fn hashes_file_and_records_size() {
239        let directory = tempfile::tempdir().unwrap();
240        let path = directory.path().join("scan.bin");
241        std::fs::write(&path, b"hello").unwrap();
242
243        let receipt = FileReceipt::from_path(ReceiptRole::Input, &path).unwrap();
244        assert_eq!(receipt.size_bytes, Some(5));
245        assert_eq!(
246            receipt.sha256.as_deref(),
247            Some("2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824")
248        );
249    }
250
251    #[test]
252    fn serializes_local_and_uri_entries() {
253        let directory = tempfile::tempdir().unwrap();
254        let path = directory.path().join("scan.bin");
255        std::fs::write(&path, b"data").unwrap();
256
257        let mut manifest = DatasetManifest::new();
258        manifest.add_file(ReceiptRole::Input, &path).unwrap();
259        manifest.add_uri(ReceiptRole::Auxiliary, "https://example.test/scan.copc.laz");
260        let json = manifest.to_json().unwrap();
261        let decoded: DatasetManifest = serde_json::from_str(&json).unwrap();
262
263        assert_eq!(decoded.version, DATASET_MANIFEST_VERSION);
264        assert_eq!(decoded.entries.len(), 2);
265        assert_eq!(decoded.entries[0].role, ReceiptRole::Input);
266        assert_eq!(decoded.entries[0].size_bytes, Some(4));
267        assert_eq!(decoded.entries[1].sha256, None);
268        let validation = decoded.validate_local_files().unwrap();
269        assert_eq!(validation.checked_local_files, 1);
270        assert_eq!(validation.uri_entries, 1);
271        assert_eq!(validation.total_bytes, 4);
272
273        let manifest_path = directory.path().join("manifest.json");
274        decoded.write_json(&manifest_path).unwrap();
275        assert_eq!(DatasetManifest::read_json(&manifest_path).unwrap(), decoded);
276    }
277
278    #[test]
279    fn rejects_changed_local_file() {
280        let directory = tempfile::tempdir().unwrap();
281        let path = directory.path().join("scan.bin");
282        std::fs::write(&path, b"before").unwrap();
283        let mut manifest = DatasetManifest::new();
284        manifest.add_file(ReceiptRole::Input, &path).unwrap();
285        std::fs::write(&path, b"after").unwrap();
286
287        let error = manifest.validate_local_files().unwrap_err();
288        assert!(error.to_string().contains("mismatch"));
289    }
290
291    #[test]
292    fn rejects_partial_or_unchecked_local_receipts() {
293        let path = std::path::PathBuf::from("/tmp/scan.bin");
294        let partial = DatasetManifest {
295            version: DATASET_MANIFEST_VERSION,
296            entries: vec![FileReceipt {
297                role: ReceiptRole::Input,
298                path: path.clone(),
299                size_bytes: Some(1),
300                sha256: None,
301            }],
302        };
303        assert!(partial.validate_local_files().unwrap_err().to_string().contains("partial"));
304
305        let unchecked = DatasetManifest {
306            version: DATASET_MANIFEST_VERSION,
307            entries: vec![FileReceipt {
308                role: ReceiptRole::Input,
309                path,
310                size_bytes: None,
311                sha256: None,
312            }],
313        };
314        assert!(unchecked.validate_local_files().unwrap_err().to_string().contains("not a URI"));
315    }
316}
spatialrust_io/manifest.rs

spatialrust_io/
manifest.rs
