Skip to content

Commit e9c405b

Browse files
fix: replace cache marker with manifest-based metadata
Signed-off-by: Andrew Paprotsky <apaprotskyi@nvidia.com>
1 parent eb4e8ec commit e9c405b

File tree

4 files changed

+1494
-1128
lines changed

4 files changed

+1494
-1128
lines changed

Cargo.lock

Lines changed: 12 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

modelexpress_common/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ google-cloud-storage = { workspace = true }
3535
rustls = { workspace = true }
3636
crc32c = { workspace = true }
3737
futures = { workspace = true }
38+
fd-lock = "4.0.4"
3839

3940
[dev-dependencies]
4041
mockall = "0.13"

modelexpress_common/src/cache.rs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -560,11 +560,18 @@ mod tests {
560560
.expect("Failed to resolve GCS path");
561561
fs::create_dir_all(gcs_model_dir.join("weights"))
562562
.expect("Failed to create GCS model directory");
563-
fs::write(gcs_model_dir.join(".mx-model"), []).expect("Failed to write GCS marker");
564563
fs::write(gcs_model_dir.join("tokenizer.json"), b"{}")
565564
.expect("Failed to write GCS tokenizer");
566565
fs::write(gcs_model_dir.join("weights/model.bin"), b"abcd")
567566
.expect("Failed to write GCS weights");
567+
let gcs_metadata_dir = gcs_model_dir.join(".mx");
568+
fs::create_dir_all(&gcs_metadata_dir).expect("Failed to create GCS metadata directory");
569+
fs::write(
570+
gcs_metadata_dir.join("manifest.json"),
571+
r#"{"version":1,"model":"gs://envbucket/dev/bake/qwen/rev123","files":[{"path":"tokenizer.json","size":2,"crc32c":"00000000","generation":null},{"path":"weights/model.bin","size":4,"crc32c":"00000000","generation":null}]}
572+
"#,
573+
)
574+
.expect("Failed to write GCS manifest");
568575

569576
let ignored_dir = cache_path.join("tmp");
570577
fs::create_dir_all(&ignored_dir).expect("Failed to create ignored directory");

0 commit comments

Comments
 (0)