-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathlib.rs
More file actions
177 lines (157 loc) · 6.26 KB
/
Copy pathlib.rs
File metadata and controls
177 lines (157 loc) · 6.26 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
mod datastore;
mod download;
mod error;
mod manager;
mod store;
use napi::threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode};
use napi_derive::napi;
use std::sync::Arc;
pub use manager::DownloadManager;
/// Reads a JSON "database" file whole. Returns `null` for a missing file —
/// the TS caller (`app/src/json-store.ts`) treats that exactly like an
/// `ENOENT` from `fs.readFileSync` — and throws for any other I/O failure so
/// it gets logged rather than silently treated as "file doesn't exist".
#[napi]
pub fn read_json_file_native(path: String) -> napi::Result<Option<String>> {
datastore::read_json_file(&path).map_err(|err| napi::Error::from_reason(err.to_string()))
}
/// Atomically writes `contents` to `path` (temp file + rename). See
/// `datastore::write_json_file_atomic` for why this must match
/// `json-store.ts`'s `writeJson` byte-for-byte.
#[napi]
pub fn write_json_file_atomic_native(path: String, contents: String) -> napi::Result<()> {
datastore::write_json_file_atomic(&path, &contents)
.map_err(|err| napi::Error::from_reason(err.to_string()))
}
/// SHA-256 hex digest — backs `audit-log-store.ts`'s hash-chain, which
/// otherwise hashes every audit event on every write.
#[napi]
pub fn sha256_hex_native(input: String) -> String {
datastore::sha256_hex(&input)
}
/// O(1) append onto an existing JSON array file — see
/// `datastore::append_json_array_element` for the full rationale. Returns
/// `false` (not an error) whenever the fast path doesn't apply, so the
/// caller falls back to a full rewrite.
#[napi]
pub fn append_json_array_element_native(path: String, element_json: String) -> napi::Result<bool> {
datastore::append_json_array_element(&path, &element_json)
.map_err(|err| napi::Error::from_reason(err.to_string()))
}
#[napi(object)]
#[derive(Clone, Copy)]
pub struct DownloadProgress {
pub received_bytes: f64,
pub total_bytes: Option<f64>,
}
/// Downloads a GGUF file from Hugging Face into `dest_path`, resuming an
/// interrupted attempt where possible, using parallel Range-request
/// connections when the server and file size support it (falling back to a
/// single stream otherwise). Mirrors the signature and observable behavior
/// of the TypeScript `downloadGgufFile` it replaces, so callers don't need
/// to change.
///
/// `expected_sha256`, when given, is checked the same way the job-based
/// `DownloadManager` already checks each shard's (see `download::job`'s
/// `run_job_with`) — after the transfer completes, before the caller can
/// treat the file as trustworthy. A mismatch deletes the finished file
/// rather than leaving corrupt bytes behind under a name that looks like a
/// successful download.
#[napi]
pub async fn download_gguf_file(
model_id: String,
filename: String,
dest_path: String,
token: Option<String>,
expected_sha256: Option<String>,
on_progress: ThreadsafeFunction<DownloadProgress>,
) -> napi::Result<()> {
let on_progress = Arc::new(on_progress);
let progress_fn: download::ProgressFn = Arc::new(move |received_bytes, total_bytes| {
let _ = on_progress.call(
Ok(DownloadProgress {
received_bytes: received_bytes as f64,
total_bytes: total_bytes.map(|t| t as f64),
}),
ThreadsafeFunctionCallMode::NonBlocking,
);
});
let url = download::build_resolve_url(&model_id, &filename).map_err(napi::Error::from)?;
download::run(
url,
filename.clone(),
dest_path.clone(),
token,
progress_fn,
download::DownloadControls::default(),
)
.await
.map_err(napi::Error::from)?;
verify_or_cleanup(&dest_path, &filename, expected_sha256.as_deref())
.await
.map_err(napi::Error::from)
}
/// Extracted from `download_gguf_file` so it's testable with plain
/// `cargo test` — unlike that function, this takes no `ThreadsafeFunction`,
/// which needs a live JS environment to construct.
async fn verify_or_cleanup(
dest_path: &str,
filename: &str,
expected_sha256: Option<&str>,
) -> Result<(), crate::error::DownloadError> {
let Some(expected) = expected_sha256 else {
return Ok(());
};
if let Err(e) =
download::verify::verify_sha256(std::path::Path::new(dest_path), filename, expected).await
{
// Corrupt bytes, not an incomplete transfer — same treatment
// download::job::run_job_with gives a checksum-mismatched shard: the
// finished file isn't trustworthy and must not be left around to be
// mistaken for a real, usable model.
tokio::fs::remove_file(dest_path).await.ok();
return Err(e);
}
Ok(())
}
#[cfg(test)]
mod verify_or_cleanup_tests {
use super::verify_or_cleanup;
use tempfile::tempdir;
#[tokio::test]
async fn no_expected_hash_skips_verification_and_keeps_the_file() {
let dir = tempdir().unwrap();
let path = dir.path().join("model.gguf");
std::fs::write(&path, b"hello world").unwrap();
verify_or_cleanup(path.to_str().unwrap(), "model.gguf", None)
.await
.expect("no expected hash means nothing to check");
assert!(path.exists());
}
#[tokio::test]
async fn matching_hash_succeeds_and_keeps_the_file() {
let dir = tempdir().unwrap();
let path = dir.path().join("model.gguf");
std::fs::write(&path, b"hello world").unwrap();
let expected = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9";
verify_or_cleanup(path.to_str().unwrap(), "model.gguf", Some(expected))
.await
.expect("hash matches");
assert!(path.exists());
}
#[tokio::test]
async fn mismatched_hash_fails_and_deletes_the_file() {
let dir = tempdir().unwrap();
let path = dir.path().join("model.gguf");
std::fs::write(&path, b"hello world").unwrap();
let err = verify_or_cleanup(
path.to_str().unwrap(),
"model.gguf",
Some("0000000000000000000000000000000000000000000000000000000000000000"),
)
.await
.expect_err("hash should not match");
assert_eq!(err.kind(), "verification_failed");
assert!(!path.exists(), "a corrupt result must not be left on disk");
}
}