From 9991f56b4c689149257dbe86357cbcab8fa228e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kat=20March=C3=A1n?= Date: Sun, 19 Feb 2023 01:18:29 -0800 Subject: [PATCH] feat(index): Add support for raw index metadata and expose index functions --- src/index.rs | 31 +++++++++++++++++++++++++++---- src/lib.rs | 4 ++-- src/put.rs | 7 +++++++ 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/index.rs b/src/index.rs index 5e2e25e..e5e73e3 100644 --- a/src/index.rs +++ b/src/index.rs @@ -1,3 +1,5 @@ +//! Raw access to the cache index. Use with caution! + use std::collections::HashSet; use std::fs::{self, OpenOptions}; use std::hash::{Hash, Hasher}; @@ -34,6 +36,8 @@ pub struct Metadata { pub size: usize, /// Arbitrary JSON associated with this entry. pub metadata: Value, + /// Raw metadata in binary form. Can be different from JSON metadata. + pub raw_metadata: Option>, } #[derive(Deserialize, Serialize, Debug)] @@ -43,6 +47,7 @@ struct SerializableMetadata { time: u128, size: usize, metadata: Value, + raw_metadata: Option>, } impl PartialEq for SerializableMetadata { @@ -59,6 +64,7 @@ impl Hash for SerializableMetadata { } } +/// Raw insertion into the cache index. pub fn insert(cache: &Path, key: &str, opts: WriteOpts) -> Result { let bucket = bucket_path(cache, key); fs::create_dir_all(bucket.parent().unwrap()).with_context(|| { @@ -73,6 +79,7 @@ pub fn insert(cache: &Path, key: &str, opts: WriteOpts) -> Result { time: opts.time.unwrap_or_else(now), size: opts.size.unwrap_or(0), metadata: opts.metadata.unwrap_or(serde_json::Value::Null), + raw_metadata: opts.raw_metadata, }) .with_context(|| format!("Failed to serialize entry with key `{key}`"))?; @@ -93,6 +100,7 @@ pub fn insert(cache: &Path, key: &str, opts: WriteOpts) -> Result { .unwrap()) } +/// Asynchronous raw insertion into the cache index. pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: WriteOpts) -> Result { let bucket = bucket_path(cache, key); crate::async_lib::create_dir_all(bucket.parent().unwrap()) @@ -109,6 +117,7 @@ pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: WriteOpts) -> time: opts.time.unwrap_or_else(now), size: opts.size.unwrap_or(0), metadata: opts.metadata.unwrap_or(serde_json::Value::Null), + raw_metadata: opts.raw_metadata, }) .with_context(|| format!("Failed to serialize entry with key `{key}`"))?; @@ -132,6 +141,7 @@ pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: WriteOpts) -> .unwrap()) } +/// Raw index Metadata access. pub fn find(cache: &Path, key: &str) -> Result> { let bucket = bucket_path(cache, key); Ok(bucket_entries(&bucket) @@ -150,6 +160,7 @@ pub fn find(cache: &Path, key: &str) -> Result> { size: entry.size, time: entry.time, metadata: entry.metadata, + raw_metadata: entry.raw_metadata, }) } else { None @@ -160,6 +171,7 @@ pub fn find(cache: &Path, key: &str) -> Result> { })) } +/// Asynchronous raw index Metadata access. pub async fn find_async(cache: &Path, key: &str) -> Result> { let bucket = bucket_path(cache, key); Ok(bucket_entries_async(&bucket) @@ -179,6 +191,7 @@ pub async fn find_async(cache: &Path, key: &str) -> Result> { size: entry.size, time: entry.time, metadata: entry.metadata, + raw_metadata: entry.raw_metadata, }) } else { None @@ -189,6 +202,7 @@ pub async fn find_async(cache: &Path, key: &str) -> Result> { })) } +/// Deletes an index entry, without deleting the actual cache data entry. pub fn delete(cache: &Path, key: &str) -> Result<()> { insert( cache, @@ -199,11 +213,14 @@ pub fn delete(cache: &Path, key: &str) -> Result<()> { sri: None, time: None, metadata: None, + raw_metadata: None, }, ) .map(|_| ()) } +/// Asynchronously deletes an index entry, without deleting the actual cache +/// data entry. pub async fn delete_async(cache: &Path, key: &str) -> Result<()> { insert( cache, @@ -214,11 +231,13 @@ pub async fn delete_async(cache: &Path, key: &str) -> Result<()> { sri: None, time: None, metadata: None, + raw_metadata: None, }, ) .map(|_| ()) } +/// Lists raw index Metadata entries. pub fn ls(cache: &Path) -> impl Iterator> { let cache_path = cache.join(format!("index-v{INDEX_VERSION}")); let cloned = cache_path.clone(); @@ -258,6 +277,7 @@ pub fn ls(cache: &Path) -> impl Iterator> { time: se.time, size: se.size, metadata: se.metadata, + raw_metadata: se.raw_metadata, }) } else { None @@ -363,7 +383,7 @@ mod tests { #[cfg(feature = "tokio")] use tokio::test as async_test; - const MOCK_ENTRY: &str = "\n251d18a2b33264ea8655695fd23c88bd874cdea2c3dc9d8f9b7596717ad30fec\t{\"key\":\"hello\",\"integrity\":\"sha1-deadbeef\",\"time\":1234567,\"size\":0,\"metadata\":null}"; + const MOCK_ENTRY: &str = "\n9cbbfe2553e7c7e1773f53f0f643fdd72008faa38da53ebcb055e5e20321ae47\t{\"key\":\"hello\",\"integrity\":\"sha1-deadbeef\",\"time\":1234567,\"size\":0,\"metadata\":null,\"raw_metadata\":null}"; fn ls_entries(dir: &Path) -> Vec { let mut entries = ls(dir) @@ -417,7 +437,8 @@ mod tests { integrity: sri, time, size: 0, - metadata: json!(null) + metadata: json!(null), + raw_metadata: None, } ); } @@ -471,7 +492,8 @@ mod tests { integrity: sri, time, size: 0, - metadata: json!(null) + metadata: json!(null), + raw_metadata: None, } ); } @@ -496,7 +518,8 @@ mod tests { integrity: sri, time, size: 0, - metadata: json!(null) + metadata: json!(null), + raw_metadata: None, } ); } diff --git a/src/lib.rs b/src/lib.rs index cfd55b6..54a4ea7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -121,7 +121,7 @@ //! Ok(()) //! } //! ``` -#![warn(missing_docs, rustdoc::missing_doc_code_examples)] +#![warn(missing_docs)] #[cfg(not(any(feature = "async-std", feature = "tokio-runtime")))] compile_error!("Either feature \"async-std\" or \"tokio-runtime\" must be enabled for this crate."); @@ -136,7 +136,7 @@ mod async_lib; mod content; mod errors; -mod index; +pub mod index; mod get; mod ls; diff --git a/src/put.rs b/src/put.rs index 1a46c30..15a3d2f 100644 --- a/src/put.rs +++ b/src/put.rs @@ -239,6 +239,7 @@ pub struct WriteOpts { pub(crate) size: Option, pub(crate) time: Option, pub(crate) metadata: Option, + pub(crate) raw_metadata: Option>, } impl WriteOpts { @@ -354,6 +355,12 @@ impl WriteOpts { self } + /// Sets arbitrary additional binary metadata to associate with the index entry. + pub fn raw_metadata(mut self, metadata: Vec) -> Self { + self.raw_metadata = Some(metadata); + self + } + /// Sets the specific time in unix milliseconds to associate with this /// entry. This is usually automatically set to the write time, but can be /// useful to change for tests and such.