diff --git a/README.md b/README.md index eead088..e63e771 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,10 @@ async fn main() -> Result<(), cacache::Error> { let dir = String::from("./my-cache"); // Write some data! - cacache::put::data(&dir, "key", b"my-async-data").await?; + cacache::write(&dir, "key", b"my-async-data").await?; // Get the data back! - let data = cacache::get::data(&dir, "key").await?; + let data = cacache::read(&dir, "key").await?; assert_eq!(data, b"my-async-data"); // Clean up the data! @@ -36,11 +36,13 @@ Using [`cargo-edit`](https://crates.io/crates/cargo-edit) ## Features -- First-class async support, using [`async-std`](https://crates.io/crates/async-std) as its runtime. Sync APIs are available but secondary. +- First-class async support, using [`async-std`](https://crates.io/crates/async-std) as its runtime. Sync APIs are available but secondary +- `std::fs`-style API - Extraction by key or by content address (shasum, etc) - [Subresource Integrity](#integrity) web standard support - Multi-hash support - safely host sha1, sha512, etc, in a single cache - Automatic content deduplication +- Atomic content writes even for large data - Fault tolerance (immune to corruption, partial writes, process races, etc) - Consistency guarantees on read and write (full data verification) - Lockless, high-concurrency cache access @@ -48,6 +50,7 @@ Using [`cargo-edit`](https://crates.io/crates/cargo-edit) - Large file support - Pretty darn fast - Arbitrary metadata storage +- Cross-platform: Windows and case-(in)sensitive filesystem support - Punches nazis ## Contributing diff --git a/benches/benchmarks.rs b/benches/benchmarks.rs index 4b22b31..5492877 100644 --- a/benches/benchmarks.rs +++ b/benches/benchmarks.rs @@ -73,17 +73,17 @@ fn baseline_read_many_async(c: &mut Criterion) { }); } -fn get_data_hash_sync(c: &mut Criterion) { +fn read_hash_sync(c: &mut Criterion) { let tmp = tempfile::tempdir().unwrap(); let cache = tmp.path().to_owned(); let data = b"hello world".to_vec(); - let sri = cacache::put::data_sync(&cache, "hello", data).unwrap(); + let sri = cacache::write_sync(&cache, "hello", data).unwrap(); c.bench_function("get::data_hash_sync", move |b| { - b.iter(|| cacache::get::data_hash_sync(black_box(&cache), black_box(&sri)).unwrap()) + b.iter(|| cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap()) }); } -fn get_data_hash_many_sync(c: &mut Criterion) { +fn read_hash_many_sync(c: &mut Criterion) { let tmp = tempfile::tempdir().unwrap(); let cache = tmp.path().to_owned(); let data: Vec<_> = (0..) @@ -92,41 +92,38 @@ fn get_data_hash_many_sync(c: &mut Criterion) { .collect(); let sris: Vec<_> = data .iter() - .map(|datum| cacache::put::data_sync(&cache, "hello", datum).unwrap()) + .map(|datum| cacache::write_sync(&cache, "hello", datum).unwrap()) .collect(); c.bench_function("get::data_hash_many_sync", move |b| { b.iter(|| { for sri in sris.iter() { - cacache::get::data_hash_sync(black_box(&cache), black_box(&sri)).unwrap(); + cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap(); } }) }); } -fn get_data_sync(c: &mut Criterion) { +fn read_sync(c: &mut Criterion) { let tmp = tempfile::tempdir().unwrap(); let cache = tmp.path().to_owned(); let data = b"hello world".to_vec(); - cacache::put::data_sync(&cache, "hello", data).unwrap(); - cacache::get::data_sync(&cache, "hello").unwrap(); + cacache::write_sync(&cache, "hello", data).unwrap(); c.bench_function("get::data_sync", move |b| { - b.iter(|| { - cacache::get::data_sync(black_box(&cache), black_box(String::from("hello"))).unwrap() - }) + b.iter(|| cacache::read_sync(black_box(&cache), black_box(String::from("hello"))).unwrap()) }); } -fn get_data_hash_sync_big_data(c: &mut Criterion) { +fn read_hash_sync_big_data(c: &mut Criterion) { let tmp = tempfile::tempdir().unwrap(); let cache = tmp.path().to_owned(); let data = vec![1; 1024 * 1024 * 5]; - let sri = cacache::put::data_sync(&cache, "hello", data).unwrap(); + let sri = cacache::write_sync(&cache, "hello", data).unwrap(); c.bench_function("get_hash_big_data", move |b| { - b.iter(|| cacache::get::data_hash_sync(black_box(&cache), black_box(&sri)).unwrap()) + b.iter(|| cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap()) }); } -fn get_data_hash_many_async(c: &mut Criterion) { +fn read_hash_many_async(c: &mut Criterion) { let tmp = tempfile::tempdir().unwrap(); let cache = tmp.path().to_owned(); let data: Vec<_> = (0..) @@ -135,51 +132,45 @@ fn get_data_hash_many_async(c: &mut Criterion) { .collect(); let sris: Vec<_> = data .iter() - .map(|datum| cacache::put::data_sync(&cache, "hello", datum).unwrap()) + .map(|datum| cacache::write_sync(&cache, "hello", datum).unwrap()) .collect(); c.bench_function("get::data_hash_many", move |b| { b.iter(|| { let tasks = sris .iter() - .map(|sri| cacache::get::data_hash(black_box(&cache), black_box(&sri))); + .map(|sri| cacache::read_hash(black_box(&cache), black_box(&sri))); task::block_on(futures::future::join_all(tasks)); }) }); } -fn get_data_hash_async(c: &mut Criterion) { +fn read_hash_async(c: &mut Criterion) { let tmp = tempfile::tempdir().unwrap(); let cache = tmp.path().to_owned(); let data = b"hello world".to_vec(); - let sri = cacache::put::data_sync(&cache, "hello", data).unwrap(); + let sri = cacache::write_sync(&cache, "hello", data).unwrap(); c.bench_function("get::data_hash", move |b| { - b.iter(|| { - task::block_on(cacache::get::data_hash(black_box(&cache), black_box(&sri))).unwrap() - }) + b.iter(|| task::block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap()) }); } -fn get_data_async(c: &mut Criterion) { +fn read_async(c: &mut Criterion) { let tmp = tempfile::tempdir().unwrap(); let cache = tmp.path().to_owned(); let data = b"hello world".to_vec(); - cacache::put::data_sync(&cache, "hello", data).unwrap(); + cacache::write_sync(&cache, "hello", data).unwrap(); c.bench_function("get::data", move |b| { - b.iter(|| { - task::block_on(cacache::get::data(black_box(&cache), black_box("hello"))).unwrap() - }) + b.iter(|| task::block_on(cacache::read(black_box(&cache), black_box("hello"))).unwrap()) }); } -fn get_data_hash_async_big_data(c: &mut Criterion) { +fn read_hash_async_big_data(c: &mut Criterion) { let tmp = tempfile::tempdir().unwrap(); let cache = tmp.path().to_owned(); let data = vec![1; 1024 * 1024 * 5]; - let sri = cacache::put::data_sync(&cache, "hello", data).unwrap(); + let sri = cacache::write_sync(&cache, "hello", data).unwrap(); c.bench_function("get::data_big_data", move |b| { - b.iter(|| { - task::block_on(cacache::get::data_hash(black_box(&cache), black_box(&sri))).unwrap() - }) + b.iter(|| task::block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap()) }); } @@ -189,13 +180,13 @@ criterion_group!( baseline_read_many_sync, baseline_read_async, baseline_read_many_async, - get_data_hash_async, - get_data_hash_many_async, - get_data_hash_sync, - get_data_hash_many_sync, - get_data_async, - get_data_sync, - get_data_hash_async_big_data, - get_data_hash_sync_big_data + read_hash_async, + read_hash_many_async, + read_hash_sync, + read_hash_many_sync, + read_async, + read_sync, + read_hash_async_big_data, + read_hash_sync_big_data ); criterion_main!(benches); diff --git a/src/get.rs b/src/get.rs index 21419b4..d10c551 100644 --- a/src/get.rs +++ b/src/get.rs @@ -8,23 +8,23 @@ use futures::prelude::*; use anyhow::{Context, Result}; use ssri::{Algorithm, Integrity}; -use crate::content::read::{self, AsyncReader, Reader}; +use crate::content::read; use crate::errors::Error; -use crate::index::{self, Entry}; +use crate::index::{self, Metadata}; // --------- // Async API // --------- -/// File handle for asynchronously reading from a content entry. +/// File handle for reading data asynchronously. /// /// Make sure to call `.check()` when done reading to verify that the /// extracted data passes integrity verification. -pub struct AsyncGet { - reader: AsyncReader, +pub struct Reader { + reader: read::AsyncReader, } -impl AsyncRead for AsyncGet { +impl AsyncRead for Reader { fn poll_read( mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>, @@ -34,11 +34,15 @@ impl AsyncRead for AsyncGet { } } -impl AsyncGet { +impl Reader { /// Checks that data read from disk passes integrity checks. Returns the /// algorithm that was used verified the data. Should be called only after /// all data has been read from disk. /// + /// This check is very cheap, since most of the verification is done on + /// the fly. This simply finalizes verification, and is always + /// synchronous. + /// /// ## Example /// ```no_run /// use async_std::prelude::*; @@ -47,11 +51,11 @@ impl AsyncGet { /// /// #[async_attributes::main] /// async fn main() -> Result<()> { - /// let mut handle = cacache::get::open("./my-cache", "my-key").await?; + /// let mut fd = cacache::Reader::open("./my-cache", "my-key").await?; /// let mut str = String::new(); - /// handle.read_to_string(&mut str).await?; + /// fd.read_to_string(&mut str).await?; /// // Remember to check that the data you got was correct! - /// handle.check()?; + /// fd.check()?; /// Ok(()) /// } /// ``` @@ -60,68 +64,67 @@ impl AsyncGet { .check() .context("Cache read data verification check failed.") } -} - -/// Opens a new file handle into the cache, looking it up in the index using -/// `key`. -/// -/// ## Example -/// ```no_run -/// use async_std::prelude::*; -/// use async_attributes; -/// use anyhow::Result; -/// -/// #[async_attributes::main] -/// async fn main() -> Result<()> { -/// let mut handle = cacache::get::open("./my-cache", "my-key").await?; -/// let mut str = String::new(); -/// handle.read_to_string(&mut str).await?; -/// // Remember to check that the data you got was correct! -/// handle.check()?; -/// Ok(()) -/// } -/// ``` -pub async fn open(cache: P, key: K) -> Result -where - P: AsRef, - K: AsRef, -{ - if let Some(entry) = index::find_async(cache.as_ref(), key.as_ref()).await? { - open_hash(cache, entry.integrity).await - } else { - Err(Error::EntryNotFound( - cache.as_ref().to_path_buf(), - key.as_ref().into(), - ))? + /// Opens a new file handle into the cache, looking it up in the index using + /// `key`. + /// + /// ## Example + /// ```no_run + /// use async_std::prelude::*; + /// use async_attributes; + /// use anyhow::Result; + /// + /// #[async_attributes::main] + /// async fn main() -> Result<()> { + /// let mut fd = cacache::Reader::open("./my-cache", "my-key").await?; + /// let mut str = String::new(); + /// fd.read_to_string(&mut str).await?; + /// // Remember to check that the data you got was correct! + /// fd.check()?; + /// Ok(()) + /// } + /// ``` + pub async fn open(cache: P, key: K) -> Result + where + P: AsRef, + K: AsRef, + { + if let Some(entry) = index::find_async(cache.as_ref(), key.as_ref()).await? { + Reader::open_hash(cache, entry.integrity).await + } else { + Err(Error::EntryNotFound( + cache.as_ref().to_path_buf(), + key.as_ref().into(), + ))? + } } -} -/// Opens a new file handle into the cache, based on its integrity address. -/// -/// ## Example -/// ```no_run -/// use async_std::prelude::*; -/// use async_attributes; -/// use anyhow::Result; -/// -/// #[async_attributes::main] -/// async fn main() -> Result<()> { -/// let sri = cacache::put::data("./my-cache", "key", b"hello world").await?; -/// let mut handle = cacache::get::open_hash("./my-cache", sri).await?; -/// let mut str = String::new(); -/// handle.read_to_string(&mut str).await?; -/// // Remember to check that the data you got was correct! -/// handle.check()?; -/// Ok(()) -/// } -/// ``` -pub async fn open_hash

(cache: P, sri: Integrity) -> Result -where - P: AsRef, -{ - Ok(AsyncGet { - reader: read::open_async(cache.as_ref(), sri).await?, - }) + /// Opens a new file handle into the cache, based on its integrity address. + /// + /// ## Example + /// ```no_run + /// use async_std::prelude::*; + /// use async_attributes; + /// use anyhow::Result; + /// + /// #[async_attributes::main] + /// async fn main() -> Result<()> { + /// let sri = cacache::write("./my-cache", "key", b"hello world").await?; + /// let mut fd = cacache::Reader::open_hash("./my-cache", sri).await?; + /// let mut str = String::new(); + /// fd.read_to_string(&mut str).await?; + /// // Remember to check that the data you got was correct! + /// fd.check()?; + /// Ok(()) + /// } + /// ``` + pub async fn open_hash

(cache: P, sri: Integrity) -> Result + where + P: AsRef, + { + Ok(Reader { + reader: read::open_async(cache.as_ref(), sri).await?, + }) + } } /// Reads the entire contents of a cache file into a bytes vector, looking the @@ -135,17 +138,47 @@ where /// /// #[async_attributes::main] /// async fn main() -> Result<()> { -/// let data = cacache::get::data("./my-cache", "my-key").await?; +/// let data: Vec = cacache::read("./my-cache", "my-key").await?; /// Ok(()) /// } /// ``` -pub async fn data(cache: P, key: K) -> Result> +pub async fn read(cache: P, key: K) -> Result> where P: AsRef, K: AsRef, { if let Some(entry) = index::find_async(cache.as_ref(), key.as_ref()).await? { - data_hash(cache, &entry.integrity).await + read_hash(cache, &entry.integrity).await + } else { + Err(Error::EntryNotFound( + cache.as_ref().to_path_buf(), + key.as_ref().into(), + ))? + } +} + +/// Reads the entire contents of a cache file into a string, looking the +/// data up by key. +/// +/// ## Example +/// ```no_run +/// use async_std::prelude::*; +/// use async_attributes; +/// use anyhow::Result; +/// +/// #[async_attributes::main] +/// async fn main() -> Result<()> { +/// let str: String = cacache::read_to_string("./my-cache", "my-key").await?; +/// Ok(()) +/// } +/// ``` +pub async fn read_to_string(cache: P, key: K) -> Result +where + P: AsRef, + K: AsRef, +{ + if let Some(entry) = index::find_async(cache.as_ref(), key.as_ref()).await? { + read_hash_to_string(cache, &entry.integrity).await } else { Err(Error::EntryNotFound( cache.as_ref().to_path_buf(), @@ -165,19 +198,20 @@ where /// /// #[async_attributes::main] /// async fn main() -> Result<()> { -/// let sri = cacache::put::data("./my-cache", "my-key", b"hello").await?; -/// let data = cacache::get::data_hash("./my-cache", &sri).await?; +/// let sri = cacache::write("./my-cache", "my-key", b"hello").await?; +/// let data: Vec = cacache::read_hash("./my-cache", &sri).await?; /// Ok(()) /// } /// ``` -pub async fn data_hash

(cache: P, sri: &Integrity) -> Result> +pub async fn read_hash

(cache: P, sri: &Integrity) -> Result> where P: AsRef, { Ok(read::read_async(cache.as_ref(), sri).await?) } -/// Copies a cache entry by key to a specified location. +/// Reads the entire contents of a cache file into a string, looking the +/// data up by its content address. /// /// ## Example /// ```no_run @@ -187,7 +221,32 @@ where /// /// #[async_attributes::main] /// async fn main() -> Result<()> { -/// cacache::get::copy("./my-cache", "my-key", "./data.txt").await?; +/// let sri = cacache::write("./my-cache", "my-key", b"hello").await?; +/// let str: String = cacache::read_hash_to_string("./my-cache", &sri).await?; +/// Ok(()) +/// } +/// ``` +pub async fn read_hash_to_string

(cache: P, sri: &Integrity) -> Result +where + P: AsRef, +{ + Ok(String::from_utf8( + read::read_async(cache.as_ref(), sri).await?, + )?) +} + +/// Copies cache data to a specified location. Returns the number of bytes +/// copied. +/// +/// ## Example +/// ```no_run +/// use async_std::prelude::*; +/// use async_attributes; +/// use anyhow::Result; +/// +/// #[async_attributes::main] +/// async fn main() -> Result<()> { +/// cacache::copy("./my-cache", "my-key", "./data.txt").await?; /// Ok(()) /// } /// ``` @@ -207,7 +266,8 @@ where } } -/// Copies a cache entry by integrity address to a specified location. +/// Copies a cache data by hash to a specified location. Returns the number of +/// bytes copied. /// /// ## Example /// ```no_run @@ -217,8 +277,8 @@ where /// /// #[async_attributes::main] /// async fn main() -> Result<()> { -/// let sri = cacache::put::data("./my-cache", "my-key", b"hello world").await?; -/// cacache::get::copy_hash("./my-cache", &sri, "./data.txt").await?; +/// let sri = cacache::write("./my-cache", "my-key", b"hello world").await?; +/// cacache::copy_hash("./my-cache", &sri, "./data.txt").await?; /// Ok(()) /// } /// ``` @@ -230,8 +290,12 @@ where read::copy_async(cache.as_ref(), sri, to.as_ref()).await } -/// Gets entry information and metadata for a certain key. -pub async fn entry(cache: P, key: K) -> Result> +/// Gets the metadata entry for a certain key. +/// +/// Note that the existence of a metadata entry is not a guarantee that the +/// underlying data exists, since they are stored and managed independently. +/// To verify that the underlying associated data exists, use `exists()`. +pub async fn metadata(cache: P, key: K) -> Result> where P: AsRef, K: AsRef, @@ -240,7 +304,7 @@ where } /// Returns true if the given hash exists in the cache. -pub async fn hash_exists>(cache: P, sri: &Integrity) -> bool { +pub async fn exists>(cache: P, sri: &Integrity) -> bool { read::has_content_async(cache.as_ref(), &sri) .await .is_some() @@ -250,22 +314,22 @@ pub async fn hash_exists>(cache: P, sri: &Integrity) -> bool { // Synchronous API // --------------- -/// File handle for reading from a content entry. +/// File handle for reading data synchronously. /// /// Make sure to call `get.check()` when done reading /// to verify that the extracted data passes integrity /// verification. -pub struct SyncGet { - reader: Reader, +pub struct SyncReader { + reader: read::Reader, } -impl std::io::Read for SyncGet { +impl std::io::Read for SyncReader { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { self.reader.read(buf) } } -impl SyncGet { +impl SyncReader { /// Checks that data read from disk passes integrity checks. Returns the /// algorithm that was used verified the data. Should be called only after /// all data has been read from disk. @@ -276,11 +340,11 @@ impl SyncGet { /// use std::io::Read; /// /// fn main() -> Result<()> { - /// let mut handle = cacache::get::open_sync("./my-cache", "my-key")?; + /// let mut fd = cacache::SyncReader::open("./my-cache", "my-key")?; /// let mut str = String::new(); - /// handle.read_to_string(&mut str)?; + /// fd.read_to_string(&mut str)?; /// // Remember to check that the data you got was correct! - /// handle.check()?; + /// fd.check()?; /// Ok(()) /// } /// ``` @@ -289,64 +353,64 @@ impl SyncGet { .check() .context("Cache read data verification check failed.") } -} -/// Opens a new synchronous file handle into the cache, looking it up in the -/// index using `key`. -/// -/// ## Example -/// ```no_run -/// use anyhow::Result; -/// use std::io::Read; -/// -/// fn main() -> Result<()> { -/// let mut handle = cacache::get::open_sync("./my-cache", "my-key")?; -/// let mut str = String::new(); -/// handle.read_to_string(&mut str)?; -/// // Remember to check that the data you got was correct! -/// handle.check()?; -/// Ok(()) -/// } -/// ``` -pub fn open_sync(cache: P, key: K) -> Result -where - P: AsRef, - K: AsRef, -{ - if let Some(entry) = index::find(cache.as_ref(), key.as_ref())? { - open_hash_sync(cache, entry.integrity) - } else { - Err(Error::EntryNotFound( - cache.as_ref().to_path_buf(), - key.as_ref().into(), - ))? + /// Opens a new synchronous file handle into the cache, looking it up in the + /// index using `key`. + /// + /// ## Example + /// ```no_run + /// use anyhow::Result; + /// use std::io::Read; + /// + /// fn main() -> Result<()> { + /// let mut fd = cacache::SyncReader::open("./my-cache", "my-key")?; + /// let mut str = String::new(); + /// fd.read_to_string(&mut str)?; + /// // Remember to check that the data you got was correct! + /// fd.check()?; + /// Ok(()) + /// } + /// ``` + pub fn open(cache: P, key: K) -> Result + where + P: AsRef, + K: AsRef, + { + if let Some(entry) = index::find(cache.as_ref(), key.as_ref())? { + SyncReader::open_hash(cache, entry.integrity) + } else { + Err(Error::EntryNotFound( + cache.as_ref().to_path_buf(), + key.as_ref().into(), + ))? + } } -} -/// Opens a new synchronous file handle into the cache, based on its integrity address. -/// -/// ## Example -/// ```no_run -/// use anyhow::Result; -/// use std::io::Read; -/// -/// fn main() -> Result<()> { -/// let sri = cacache::put::data_sync("./my-cache", "key", b"hello world")?; -/// let mut handle = cacache::get::open_hash_sync("./my-cache", sri)?; -/// let mut str = String::new(); -/// handle.read_to_string(&mut str)?; -/// // Remember to check that the data you got was correct! -/// handle.check()?; -/// Ok(()) -/// } -/// ``` -pub fn open_hash_sync

(cache: P, sri: Integrity) -> Result -where - P: AsRef, -{ - Ok(SyncGet { - reader: read::open(cache.as_ref(), sri)?, - }) + /// Opens a new synchronous file handle into the cache, based on its integrity address. + /// + /// ## Example + /// ```no_run + /// use anyhow::Result; + /// use std::io::Read; + /// + /// fn main() -> Result<()> { + /// let sri = cacache::write_sync("./my-cache", "key", b"hello world")?; + /// let mut fd = cacache::SyncReader::open_hash("./my-cache", sri)?; + /// let mut str = String::new(); + /// fd.read_to_string(&mut str)?; + /// // Remember to check that the data you got was correct! + /// fd.check()?; + /// Ok(()) + /// } + /// ``` + pub fn open_hash

(cache: P, sri: Integrity) -> Result + where + P: AsRef, + { + Ok(SyncReader { + reader: read::open(cache.as_ref(), sri)?, + }) + } } /// Reads the entire contents of a cache file synchronously into a bytes @@ -358,17 +422,17 @@ where /// use std::io::Read; /// /// fn main() -> Result<()> { -/// let data = cacache::get::data_sync("./my-cache", "my-key")?; +/// let data = cacache::read_sync("./my-cache", "my-key")?; /// Ok(()) /// } /// ``` -pub fn data_sync(cache: P, key: K) -> Result> +pub fn read_sync(cache: P, key: K) -> Result> where P: AsRef, K: AsRef, { if let Some(entry) = index::find(cache.as_ref(), key.as_ref())? { - data_hash_sync(cache, &entry.integrity) + read_hash_sync(cache, &entry.integrity) } else { Err(Error::EntryNotFound( cache.as_ref().to_path_buf(), @@ -377,6 +441,33 @@ where } } +/// Reads the entire contents of a cache file synchronously into a string, +/// looking the data up by key. +/// +/// ## Example +/// ```no_run +/// use anyhow::Result; +/// use std::io::Read; +/// +/// fn main() -> Result<()> { +/// let str: String = cacache::read_to_string_sync("./my-cache", "my-key")?; +/// Ok(()) +/// } +/// ``` +pub fn read_to_string_sync(cache: P, key: K) -> Result +where + P: AsRef, + K: AsRef, +{ + if let Some(entry) = index::find(cache.as_ref(), key.as_ref())? { + read_hash_to_string_sync(cache, &entry.integrity) + } else { + Err(Error::EntryNotFound( + cache.as_ref().to_path_buf(), + key.as_ref().into(), + ))? + } +} /// Reads the entire contents of a cache file synchronously into a bytes /// vector, looking the data up by its content address. /// @@ -386,19 +477,20 @@ where /// use std::io::Read; /// /// fn main() -> Result<()> { -/// let sri = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; -/// let data = cacache::get::data_hash_sync("./my-cache", &sri)?; +/// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?; +/// let data = cacache::read_hash_sync("./my-cache", &sri)?; /// Ok(()) /// } /// ``` -pub fn data_hash_sync

(cache: P, sri: &Integrity) -> Result> +pub fn read_hash_sync

(cache: P, sri: &Integrity) -> Result> where P: AsRef, { Ok(read::read(cache.as_ref(), sri)?) } -/// Copies a cache entry by key to a specified location. +/// Reads the entire contents of a cache file synchronously into a string, +/// looking the data up by its content address. /// /// ## Example /// ```no_run @@ -406,7 +498,28 @@ where /// use std::io::Read; /// /// fn main() -> Result<()> { -/// cacache::get::copy_sync("./my-cache", "my-key", "./my-hello.txt")?; +/// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?; +/// let data = cacache::read_hash_sync("./my-cache", &sri)?; +/// Ok(()) +/// } +/// ``` +pub fn read_hash_to_string_sync

(cache: P, sri: &Integrity) -> Result +where + P: AsRef, +{ + Ok(String::from_utf8(read::read(cache.as_ref(), sri)?)?) +} + +/// Copies a cache entry by key to a specified location. Returns the number of +/// bytes copied. +/// +/// ## Example +/// ```no_run +/// use anyhow::Result; +/// use std::io::Read; +/// +/// fn main() -> Result<()> { +/// cacache::copy_sync("./my-cache", "my-key", "./my-hello.txt")?; /// Ok(()) /// } /// ``` @@ -426,7 +539,8 @@ where } } -/// Copies a cache entry by integrity address to a specified location. +/// Copies a cache entry by integrity address to a specified location. Returns +/// the number of bytes copied. /// /// ## Example /// ```no_run @@ -434,8 +548,8 @@ where /// use std::io::Read; /// /// fn main() -> Result<()> { -/// let sri = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; -/// cacache::get::copy_hash_sync("./my-cache", &sri, "./my-hello.txt")?; +/// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?; +/// cacache::copy_hash_sync("./my-cache", &sri, "./my-hello.txt")?; /// Ok(()) /// } /// ``` @@ -447,8 +561,12 @@ where read::copy(cache.as_ref(), sri, to.as_ref()) } -/// Gets entry information and metadata for a certain key. -pub fn entry_sync(cache: P, key: K) -> Result> +/// Gets metadata for a certain key. +/// +/// Note that the existence of a metadata entry is not a guarantee that the +/// underlying data exists, since they are stored and managed independently. +/// To verify that the underlying associated data exists, use `exists_sync()`. +pub fn metadata_sync(cache: P, key: K) -> Result> where P: AsRef, K: AsRef, @@ -457,49 +575,42 @@ where } /// Returns true if the given hash exists in the cache. -pub fn hash_exists_sync>(cache: P, sri: &Integrity) -> bool { +pub fn exists_sync>(cache: P, sri: &Integrity) -> bool { read::has_content(cache.as_ref(), &sri).is_some() } #[cfg(test)] mod tests { + use async_attributes; + use async_std::fs as afs; use async_std::prelude::*; - use async_std::{fs as afs, task}; use std::fs; use tempfile; - #[test] - fn test_open() { - task::block_on(async { - let tmp = tempfile::tempdir().unwrap(); - let dir = tmp.path().to_owned(); - crate::put::data(&dir, "my-key", b"hello world") - .await - .unwrap(); + #[async_attributes::test] + async fn test_open() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_owned(); + crate::write(&dir, "my-key", b"hello world").await.unwrap(); - let mut handle = crate::get::open(&dir, "my-key").await.unwrap(); - let mut str = String::new(); - handle.read_to_string(&mut str).await.unwrap(); - handle.check().unwrap(); - assert_eq!(str, String::from("hello world")); - }); + let mut handle = crate::Reader::open(&dir, "my-key").await.unwrap(); + let mut str = String::new(); + handle.read_to_string(&mut str).await.unwrap(); + handle.check().unwrap(); + assert_eq!(str, String::from("hello world")); } - #[test] - fn test_open_hash() { - task::block_on(async { - let tmp = tempfile::tempdir().unwrap(); - let dir = tmp.path().to_owned(); - let sri = crate::put::data(&dir, "my-key", b"hello world") - .await - .unwrap(); + #[async_attributes::test] + async fn test_open_hash() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_owned(); + let sri = crate::write(&dir, "my-key", b"hello world").await.unwrap(); - let mut handle = crate::get::open_hash(&dir, sri).await.unwrap(); - let mut str = String::new(); - handle.read_to_string(&mut str).await.unwrap(); - handle.check().unwrap(); - assert_eq!(str, String::from("hello world")); - }); + let mut handle = crate::Reader::open_hash(&dir, sri).await.unwrap(); + let mut str = String::new(); + handle.read_to_string(&mut str).await.unwrap(); + handle.check().unwrap(); + assert_eq!(str, String::from("hello world")); } #[test] @@ -507,9 +618,9 @@ mod tests { use std::io::prelude::*; let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); + crate::write_sync(&dir, "my-key", b"hello world").unwrap(); - let mut handle = crate::get::open_sync(&dir, "my-key").unwrap(); + let mut handle = crate::SyncReader::open(&dir, "my-key").unwrap(); let mut str = String::new(); handle.read_to_string(&mut str).unwrap(); handle.check().unwrap(); @@ -521,93 +632,117 @@ mod tests { use std::io::prelude::*; let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - let sri = crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); + let sri = crate::write_sync(&dir, "my-key", b"hello world").unwrap(); - let mut handle = crate::get::open_hash_sync(&dir, sri).unwrap(); + let mut handle = crate::SyncReader::open_hash(&dir, sri).unwrap(); let mut str = String::new(); handle.read_to_string(&mut str).unwrap(); handle.check().unwrap(); assert_eq!(str, String::from("hello world")); } - #[test] - fn test_data() { - task::block_on(async { - let tmp = tempfile::tempdir().unwrap(); - let dir = tmp.path().to_owned(); - crate::put::data(&dir, "my-key", b"hello world") - .await - .unwrap(); - - let data = crate::get::data(&dir, "my-key").await.unwrap(); - assert_eq!(data, b"hello world"); - }); - } - - #[test] - fn test_data_hash() { - task::block_on(async { - let tmp = tempfile::tempdir().unwrap(); - let dir = tmp.path().to_owned(); - let sri = crate::put::data(&dir, "my-key", b"hello world") - .await - .unwrap(); - - let data = crate::get::data_hash(&dir, &sri).await.unwrap(); - assert_eq!(data, b"hello world"); - }); - } - - #[test] - fn test_data_sync() { + #[async_attributes::test] + async fn test_read() { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); + crate::write(&dir, "my-key", b"hello world").await.unwrap(); - let data = crate::get::data_sync(&dir, "my-key").unwrap(); + let data = crate::read(&dir, "my-key").await.unwrap(); + assert_eq!(data, b"hello world"); + } + + #[async_attributes::test] + async fn test_read_to_string() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_owned(); + crate::write(&dir, "my-key", "hello world").await.unwrap(); + + let data = crate::read_to_string(&dir, "my-key").await.unwrap(); + assert_eq!(data, "hello world"); + } + + #[async_attributes::test] + async fn test_read_hash() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_owned(); + let sri = crate::write(&dir, "my-key", b"hello world").await.unwrap(); + + let data = crate::read_hash(&dir, &sri).await.unwrap(); + assert_eq!(data, b"hello world"); + } + + #[async_attributes::test] + async fn test_read_hash_to_string() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_owned(); + let sri = crate::write(&dir, "my-key", "hello world").await.unwrap(); + + let data = crate::read_hash_to_string(&dir, &sri).await.unwrap(); + assert_eq!(data, "hello world"); + } + + #[test] + fn test_read_sync() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_owned(); + crate::write_sync(&dir, "my-key", b"hello world").unwrap(); + + let data = crate::read_sync(&dir, "my-key").unwrap(); assert_eq!(data, b"hello world"); } #[test] - fn test_data_hash_sync() { + fn test_read_to_string_sync() { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - let sri = crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); + crate::write_sync(&dir, "my-key", "hello world").unwrap(); - let data = crate::get::data_hash_sync(&dir, &sri).unwrap(); + let data = crate::read_to_string_sync(&dir, "my-key").unwrap(); + assert_eq!(data, "hello world"); + } + + #[test] + fn test_read_hash_sync() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_owned(); + let sri = crate::write_sync(&dir, "my-key", b"hello world").unwrap(); + + let data = crate::read_hash_sync(&dir, &sri).unwrap(); assert_eq!(data, b"hello world"); } #[test] - fn test_copy() { - task::block_on(async { - let tmp = tempfile::tempdir().unwrap(); - let dir = tmp.path(); - let dest = dir.join("data"); - crate::put::data(&dir, "my-key", b"hello world") - .await - .unwrap(); + fn test_read_hash_to_string_sync() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_owned(); + let sri = crate::write_sync(&dir, "my-key", "hello world").unwrap(); - crate::get::copy(&dir, "my-key", &dest).await.unwrap(); - let data = afs::read(&dest).await.unwrap(); - assert_eq!(data, b"hello world"); - }); + let data = crate::read_hash_to_string_sync(&dir, &sri).unwrap(); + assert_eq!(data, "hello world"); } - #[test] - fn test_copy_hash() { - task::block_on(async { - let tmp = tempfile::tempdir().unwrap(); - let dir = tmp.path(); - let dest = dir.join("data"); - let sri = crate::put::data(&dir, "my-key", b"hello world") - .await - .unwrap(); + #[async_attributes::test] + async fn test_copy() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let dest = dir.join("data"); + crate::write(&dir, "my-key", b"hello world").await.unwrap(); - crate::get::copy_hash(&dir, &sri, &dest).await.unwrap(); - let data = afs::read(&dest).await.unwrap(); - assert_eq!(data, b"hello world"); - }); + crate::copy(&dir, "my-key", &dest).await.unwrap(); + let data = afs::read(&dest).await.unwrap(); + assert_eq!(data, b"hello world"); + } + + #[async_attributes::test] + async fn test_copy_hash() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let dest = dir.join("data"); + let sri = crate::write(&dir, "my-key", b"hello world").await.unwrap(); + + crate::copy_hash(&dir, &sri, &dest).await.unwrap(); + let data = afs::read(&dest).await.unwrap(); + assert_eq!(data, b"hello world"); } #[test] @@ -615,9 +750,9 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path(); let dest = dir.join("data"); - crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); + crate::write_sync(&dir, "my-key", b"hello world").unwrap(); - crate::get::copy_sync(&dir, "my-key", &dest).unwrap(); + crate::copy_sync(&dir, "my-key", &dest).unwrap(); let data = fs::read(&dest).unwrap(); assert_eq!(data, b"hello world"); } @@ -627,9 +762,9 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path(); let dest = dir.join("data"); - let sri = crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); + let sri = crate::write_sync(&dir, "my-key", b"hello world").unwrap(); - crate::get::copy_hash_sync(&dir, &sri, &dest).unwrap(); + crate::copy_hash_sync(&dir, &sri, &dest).unwrap(); let data = fs::read(&dest).unwrap(); assert_eq!(data, b"hello world"); } diff --git a/src/index.rs b/src/index.rs index e5f2f34..58643f7 100644 --- a/src/index.rs +++ b/src/index.rs @@ -21,13 +21,13 @@ use sha2::Sha256; use ssri::Integrity; use walkdir::WalkDir; -use crate::put::PutOpts; +use crate::put::WriteOpts; const INDEX_VERSION: &str = "5"; /// Represents a cache index entry, which points to content. #[derive(PartialEq, Debug)] -pub struct Entry { +pub struct Metadata { /// Key this entry is stored under. pub key: String, /// Integrity hash for the stored data. Acts as a key into {cache}/content. @@ -36,12 +36,12 @@ pub struct Entry { pub time: u128, /// Size of data associated with this entry. pub size: usize, - /// Arbitrary JSON metadata associated with this entry. + /// Arbitrary JSON associated with this entry. pub metadata: Value, } #[derive(Deserialize, Serialize, Debug)] -struct SerializableEntry { +struct SerializableMetadata { key: String, integrity: Option, time: u128, @@ -49,21 +49,21 @@ struct SerializableEntry { metadata: Value, } -impl PartialEq for SerializableEntry { +impl PartialEq for SerializableMetadata { fn eq(&self, other: &Self) -> bool { self.key == other.key } } -impl Eq for SerializableEntry {} +impl Eq for SerializableMetadata {} -impl Hash for SerializableEntry { +impl Hash for SerializableMetadata { fn hash(&self, state: &mut H) { self.key.hash(state); } } -pub fn insert(cache: &Path, key: &str, opts: PutOpts) -> Result { +pub fn insert(cache: &Path, key: &str, opts: WriteOpts) -> Result { let bucket = bucket_path(&cache, &key); #[cfg(unix)] { @@ -84,7 +84,7 @@ pub fn insert(cache: &Path, key: &str, opts: PutOpts) -> Result { bucket.parent().unwrap() ) })?; - let stringified = serde_json::to_string(&SerializableEntry { + let stringified = serde_json::to_string(&SerializableMetadata { key: key.to_owned(), integrity: opts.sri.clone().map(|x| x.to_string()), time: opts.time.unwrap_or_else(now), @@ -112,11 +112,11 @@ pub fn insert(cache: &Path, key: &str, opts: PutOpts) -> Result { .unwrap()) } -pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: PutOpts) -> Result { +pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: WriteOpts) -> Result { let bucket = bucket_path(&cache, &key); let tmpbucket = bucket.clone(); #[cfg(unix)] - let PutOpts { uid, gid, .. } = opts; + let WriteOpts { uid, gid, .. } = opts; task::spawn_blocking(move || { let parent = tmpbucket.parent().unwrap(); #[cfg(unix)] @@ -138,7 +138,7 @@ pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: PutOpts) -> R Ok::<(), anyhow::Error>(()) }) .await?; - let stringified = serde_json::to_string(&SerializableEntry { + let stringified = serde_json::to_string(&SerializableMetadata { key: key.to_owned(), integrity: opts.sri.clone().map(|x| x.to_string()), time: opts.time.unwrap_or_else(now), @@ -168,7 +168,7 @@ pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: PutOpts) -> R .unwrap()) } -pub fn find(cache: &Path, key: &str) -> Result> { +pub fn find(cache: &Path, key: &str) -> Result> { let bucket = bucket_path(cache, &key); Ok(bucket_entries(&bucket) .with_context(|| format!("Failed to read index bucket entries from {:?}", bucket))? @@ -180,7 +180,7 @@ pub fn find(cache: &Path, key: &str) -> Result> { Ok(sri) => sri, _ => return acc, }; - Some(Entry { + Some(Metadata { key: entry.key, integrity, size: entry.size, @@ -196,7 +196,7 @@ pub fn find(cache: &Path, key: &str) -> Result> { })) } -pub async fn find_async(cache: &Path, key: &str) -> Result> { +pub async fn find_async(cache: &Path, key: &str) -> Result> { let bucket = bucket_path(cache, &key); Ok(bucket_entries_async(&bucket) .await @@ -209,7 +209,7 @@ pub async fn find_async(cache: &Path, key: &str) -> Result> { Ok(sri) => sri, _ => return acc, }; - Some(Entry { + Some(Metadata { key: entry.key, integrity, size: entry.size, @@ -229,7 +229,7 @@ pub fn delete(cache: &Path, key: &str) -> Result<()> { insert( cache, key, - PutOpts { + WriteOpts { algorithm: None, size: None, sri: None, @@ -248,7 +248,7 @@ pub async fn delete_async(cache: &Path, key: &str) -> Result<()> { insert( cache, key, - PutOpts { + WriteOpts { algorithm: None, size: None, sri: None, @@ -263,7 +263,7 @@ pub async fn delete_async(cache: &Path, key: &str) -> Result<()> { .map(|_| ()) } -pub fn ls(cache: &Path) -> impl Iterator> { +pub fn ls(cache: &Path) -> impl Iterator> { WalkDir::new(cache.join(format!("index-v{}", INDEX_VERSION))) .into_iter() .map(|bucket| { @@ -274,11 +274,11 @@ pub fn ls(cache: &Path) -> impl Iterator> { Ok(bucket_entries(bucket.path())? .into_iter() - .collect::>() + .collect::>() .into_iter() .filter_map(|se| { if let Some(i) = se.integrity { - Some(Entry { + Some(Metadata { key: se.key, integrity: i.parse().unwrap(), time: se.time, @@ -325,7 +325,7 @@ fn now() -> u128 { .as_millis() } -fn bucket_entries(bucket: &Path) -> Result> { +fn bucket_entries(bucket: &Path) -> Result> { use std::io::{BufRead, BufReader}; fs::File::open(bucket) .map(|file| { @@ -338,7 +338,7 @@ fn bucket_entries(bucket: &Path) -> Result> { // Something's wrong with the entry. Abort. _ => return None, }; - serde_json::from_str::(entry_str).ok() + serde_json::from_str::(entry_str).ok() }) .collect() }) @@ -351,7 +351,7 @@ fn bucket_entries(bucket: &Path) -> Result> { }) } -async fn bucket_entries_async(bucket: &Path) -> Result> { +async fn bucket_entries_async(bucket: &Path) -> Result> { use async_std::io::BufReader; use futures::io::AsyncBufReadExt; use futures::stream::StreamExt; @@ -374,7 +374,7 @@ async fn bucket_entries_async(bucket: &Path) -> Result> { // Something's wrong with the entry. Abort. _ => continue, }; - if let Ok(serialized) = serde_json::from_str::(entry_str) { + if let Ok(serialized) = serde_json::from_str::(entry_str) { vec.push(serialized); } } @@ -396,7 +396,7 @@ mod tests { let dir = tmp.path().to_owned(); let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let time = 1_234_567; - let opts = PutOpts::new().integrity(sri).time(time); + let opts = WriteOpts::new().integrity(sri).time(time); insert(&dir, "hello", opts).unwrap(); let entry = std::fs::read_to_string(bucket_path(&dir, "hello")).unwrap(); assert_eq!(entry, MOCK_ENTRY); @@ -408,7 +408,7 @@ mod tests { let dir = tmp.path().to_owned(); let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let time = 1_234_567; - let opts = PutOpts::new().integrity(sri).time(time); + let opts = WriteOpts::new().integrity(sri).time(time); task::block_on(async { insert_async(&dir, "hello", opts).await.unwrap(); }); @@ -428,7 +428,7 @@ mod tests { let entry = find(&dir, "hello").unwrap().unwrap(); assert_eq!( entry, - Entry { + Metadata { key: String::from("hello"), integrity: sri, time, @@ -451,7 +451,7 @@ mod tests { let dir = tmp.path().to_owned(); let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let time = 1_234_567; - let opts = PutOpts::new().integrity(sri).time(time); + let opts = WriteOpts::new().integrity(sri).time(time); insert(&dir, "hello", opts).unwrap(); delete(&dir, "hello").unwrap(); assert_eq!(find(&dir, "hello").unwrap(), None); @@ -463,7 +463,7 @@ mod tests { let dir = tmp.path().to_owned(); let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let time = 1_234_567; - let opts = PutOpts::new().integrity(sri).time(time); + let opts = WriteOpts::new().integrity(sri).time(time); insert(&dir, "hello", opts).unwrap(); task::block_on(async { delete_async(&dir, "hello").await.unwrap(); @@ -477,12 +477,12 @@ mod tests { let dir = tmp.path().to_owned(); let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let time = 1_234_567; - let opts = PutOpts::new().integrity(sri.clone()).time(time); + let opts = WriteOpts::new().integrity(sri.clone()).time(time); insert(&dir, "hello", opts).unwrap(); let entry = find(&dir, "hello").unwrap().unwrap(); assert_eq!( entry, - Entry { + Metadata { key: String::from("hello"), integrity: sri, time, @@ -498,14 +498,14 @@ mod tests { let dir = tmp.path().to_owned(); let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let time = 1_234_567; - let opts = PutOpts::new().integrity(sri.clone()).time(time); + let opts = WriteOpts::new().integrity(sri.clone()).time(time); task::block_on(async { insert_async(&dir, "hello", opts).await.unwrap(); }); let entry = task::block_on(async { find_async(&dir, "hello").await.unwrap().unwrap() }); assert_eq!( entry, - Entry { + Metadata { key: String::from("hello"), integrity: sri, time, @@ -521,9 +521,9 @@ mod tests { let dir = tmp.path().to_owned(); let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let time = 1_234_567; - let opts = PutOpts::new().integrity(sri.clone()).time(time); + let opts = WriteOpts::new().integrity(sri.clone()).time(time); insert(&dir, "hello", opts).unwrap(); - let opts = PutOpts::new().integrity(sri).time(time); + let opts = WriteOpts::new().integrity(sri).time(time); insert(&dir, "world", opts).unwrap(); let mut entries = ls(&dir) diff --git a/src/lib.rs b/src/lib.rs index c291de9..7e5fb18 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,31 @@ //! caches. It's really fast, really good at concurrency, and it will never //! give you corrupted data, even if cache files get corrupted or manipulated. //! +//! ## API Layout +//! +//! The cacache API is organized roughly similar to `std::fs`; most of the +//! toplevel functionality is available as free functions directly in the +//! `cacache` module, with some additional functionality available through +//! returned objects, as well as `WriteOpts`, which is analogous to +//! `OpenOpts`, but is only able to write. +//! +//! One major difference is that the default APIs are all async functions, as +//! opposed to `std::fs`, where they're all synchronous. Synchronous APIs in +//! cacache are accessible through the `_sync` suffix. +//! +//! ### Suffixes +//! +//! You may notice various suffixes associated with otherwise familiar +//! functions: +//! +//! * `_sync` - Most cacache APIs are asynchronous by default. Anything using +//! the `_sync` suffix behaves just like its unprefixed counterpart, except +//! the operation is synchronous. +//! * `_hash` - Since cacache is a content-addressable cache, the `_hash` +//! suffix means you're interacting directly with content data, skipping the +//! index and its metadata. These functions use an `Integrity` to look up +//! data, instead of a string key. +//! //! ## Examples //! //! Un-suffixed APIs are all async, using @@ -15,10 +40,10 @@ //! #[async_attributes::main] //! async fn main() -> Result<()> { //! // Data goes in... -//! cacache::put::data("./my-cache", "key", b"hello").await?; +//! cacache::write("./my-cache", "key", b"hello").await?; //! //! // ...data comes out! -//! let data = cacache::get::data("./my-cache", "key").await?; +//! let data = cacache::read("./my-cache", "key").await?; //! assert_eq!(data, b"hello"); //! //! Ok(()) @@ -40,10 +65,10 @@ //! #[async_attributes::main] //! async fn main() -> Result<()> { //! // Data goes in... -//! let sri = cacache::put::data("./my-cache", "key", b"hello").await?; +//! let sri = cacache::write("./my-cache", "key", b"hello").await?; //! //! // ...data gets looked up by `sri` ("Subresource Integrity"). -//! let data = cacache::get::data_hash("./my-cache", &sri).await?; +//! let data = cacache::read_hash("./my-cache", &sri).await?; //! assert_eq!(data, b"hello"); //! //! Ok(()) @@ -62,15 +87,15 @@ //! //! #[async_attributes::main] //! async fn main() -> Result<()> { -//! let mut fd = cacache::put::PutOpts::new().open("./my-cache", "key").await?; +//! let mut fd = cacache::Writer::create("./my-cache", "key").await?; //! for _ in 0..10 { //! fd.write_all(b"very large data").await?; //! } -//! // Data is only persisted to the cache after you do `fd.commit()`! +//! // Data is only committed to the cache after you do `fd.commit()`! //! let sri = fd.commit().await?; //! println!("integrity: {}", &sri); //! -//! let mut fd = cacache::get::open("./my-cache", "key").await?; +//! let mut fd = cacache::Reader::open("./my-cache", "key").await?; //! let mut buf = String::new(); //! fd.read_to_string(&mut buf).await?; //! @@ -85,13 +110,17 @@ //! //! ### Sync API //! -//! There are also sync APIs available if you don't want to use async/await: +//! There are also sync APIs available if you don't want to use async/await. +//! The synchronous APIs are generally faster for linear operations -- that +//! is, doing one thing after another, as opposed to doing many things at +//! once. If you're only reading and writing one thing at a time across your +//! application, you probably want to use these instead. //! //! ```no_run //! use anyhow::Result; //! fn main() -> Result<()> { -//! cacache::put::data_sync("./my-cache", "key", b"my-data").unwrap(); -//! let data = cacache::get::data_sync("./my-cache", "key").unwrap(); +//! cacache::write_sync("./my-cache", "key", b"my-data").unwrap(); +//! let data = cacache::read_sync("./my-cache", "key").unwrap(); //! assert_eq!(data, b"my-data"); //! Ok(()) //! } @@ -105,10 +134,15 @@ mod content; mod errors; mod index; -pub mod get; -pub mod ls; -pub mod put; -pub mod rm; +mod get; +mod ls; +mod put; +mod rm; pub use errors::Error; -pub use index::Entry; +pub use index::Metadata; + +pub use get::*; +pub use ls::*; +pub use put::*; +pub use rm::*; diff --git a/src/ls.rs b/src/ls.rs index 2c22a57..90670da 100644 --- a/src/ls.rs +++ b/src/ls.rs @@ -4,6 +4,6 @@ use std::path::Path; use crate::index; /// Returns a synchronous iterator that lists all cache index entries. -pub fn all_sync>(cache: P) -> impl Iterator { +pub fn list_sync>(cache: P) -> impl Iterator { index::ls(cache.as_ref()) } diff --git a/src/put.rs b/src/put.rs index fa5cd9b..9076477 100644 --- a/src/put.rs +++ b/src/put.rs @@ -26,27 +26,17 @@ use std::task::{Context as TaskContext, Poll}; /// /// #[async_attributes::main] /// async fn main() -> Result<()> { -/// cacache::put::data("./my-cache", "my-key", b"hello").await?; +/// cacache::write("./my-cache", "my-key", b"hello").await?; /// Ok(()) /// } /// ``` -pub async fn data(cache: P, key: K, data: D) -> Result +pub async fn write(cache: P, key: K, data: D) -> Result where P: AsRef, D: AsRef<[u8]>, K: AsRef, { - let mut writer = PutOpts::new() - .algorithm(Algorithm::Sha256) - .open(cache.as_ref(), key.as_ref()) - .await - .with_context(|| { - format!( - "Failed to open a write handle for key {} for cache at {:?}", - key.as_ref(), - cache.as_ref() - ) - })?; + let mut writer = Writer::create(cache.as_ref(), key.as_ref()).await?; writer.write_all(data.as_ref()).await.with_context(|| { format!( "Failed to write to cache data for key {} for cache at {:?}", @@ -64,15 +54,15 @@ where } /// A reference to an open file writing to the cache. -pub struct AsyncPut { +pub struct Writer { cache: PathBuf, key: String, written: usize, pub(crate) writer: write::AsyncWriter, - opts: PutOpts, + opts: WriteOpts, } -impl AsyncWrite for AsyncPut { +impl AsyncWrite for Writer { fn poll_write( mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>, @@ -90,8 +80,43 @@ impl AsyncWrite for AsyncPut { } } -impl AsyncPut { - /// Closes the AsyncPut handle and writes content and index entries. Also +impl Writer { + /// Creates a new writable file handle into the cache. + /// + /// ## Example + /// ```no_run + /// use async_attributes; + /// use async_std::prelude::*; + /// use anyhow::Result; + /// + /// #[async_attributes::main] + /// async fn main() -> Result<()> { + /// let mut fd = cacache::Writer::create("./my-cache", "my-key").await?; + /// fd.write_all(b"hello world").await?; + /// // Data is not saved into the cache until you commit it. + /// fd.commit().await?; + /// Ok(()) + /// } + /// ``` + pub async fn create(cache: P, key: K) -> Result + where + P: AsRef, + K: AsRef, + { + WriteOpts::new() + .algorithm(Algorithm::Sha256) + .open(cache.as_ref(), key.as_ref()) + .await + .with_context(|| { + format!( + "Failed to open a write handle for key {} for cache at {:?}", + key.as_ref(), + cache.as_ref() + ) + }) + } + + /// Closes the Writer handle and writes content and index entries. Also /// verifies data against `size` and `integrity` options, if provided. /// Must be called manually in order to complete the writing process, /// otherwise everything will be thrown out. @@ -142,26 +167,17 @@ impl AsyncPut { /// use std::io::Read; /// /// fn main() -> Result<()> { -/// let data = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; +/// let data = cacache::write_sync("./my-cache", "my-key", b"hello")?; /// Ok(()) /// } /// ``` -pub fn data_sync(cache: P, key: K, data: D) -> Result +pub fn write_sync(cache: P, key: K, data: D) -> Result where P: AsRef, D: AsRef<[u8]>, K: AsRef, { - let mut writer = PutOpts::new() - .algorithm(Algorithm::Sha256) - .open_sync(cache.as_ref(), key.as_ref()) - .with_context(|| { - format!( - "Failed to open a write handle for key {} for cache at {:?}", - key.as_ref(), - cache.as_ref() - ) - })?; + let mut writer = SyncWriter::create(cache.as_ref(), key.as_ref())?; writer.write_all(data.as_ref()).with_context(|| { format!( "Failed to write to cache data for key {} for cache at {:?}", @@ -180,7 +196,7 @@ where /// Builder for options and flags for opening a new cache file to write data into. #[derive(Clone, Default)] -pub struct PutOpts { +pub struct WriteOpts { pub(crate) algorithm: Option, pub(crate) sri: Option, pub(crate) size: Option, @@ -192,19 +208,19 @@ pub struct PutOpts { pub(crate) gid: Option, } -impl PutOpts { +impl WriteOpts { /// Creates a blank set of cache writing options. - pub fn new() -> PutOpts { + pub fn new() -> WriteOpts { Default::default() } - /// Opens the file handle for writing, returning an AsyncPut instance. - pub async fn open(self, cache: P, key: K) -> Result + /// Opens the file handle for writing, returning an Writer instance. + pub async fn open(self, cache: P, key: K) -> Result where P: AsRef, K: AsRef, { - Ok(AsyncPut { + Ok(Writer { cache: cache.as_ref().to_path_buf(), key: String::from(key.as_ref()), written: 0, @@ -217,13 +233,13 @@ impl PutOpts { }) } - /// Opens the file handle for writing synchronously, returning a SyncPut instance. - pub fn open_sync(self, cache: P, key: K) -> Result + /// Opens the file handle for writing synchronously, returning a SyncWriter instance. + pub fn open_sync(self, cache: P, key: K) -> Result where P: AsRef, K: AsRef, { - Ok(SyncPut { + Ok(SyncWriter { cache: cache.as_ref().to_path_buf(), key: String::from(key.as_ref()), written: 0, @@ -281,15 +297,15 @@ impl PutOpts { } /// A reference to an open file writing to the cache. -pub struct SyncPut { +pub struct SyncWriter { cache: PathBuf, key: String, written: usize, pub(crate) writer: write::Writer, - opts: PutOpts, + opts: WriteOpts, } -impl Write for SyncPut { +impl Write for SyncWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { self.writer.write(buf) } @@ -298,8 +314,40 @@ impl Write for SyncPut { } } -impl SyncPut { - /// Closes the Put handle and writes content and index entries. Also +impl SyncWriter { + /// Creates a new writable file handle into the cache. + /// + /// ## Example + /// ```no_run + /// use anyhow::Result; + /// use std::io::prelude::*; + /// + /// fn main() -> Result<()> { + /// let mut fd = cacache::SyncWriter::create("./my-cache", "my-key")?; + /// fd.write_all(b"hello world")?; + /// // Data is not saved into the cache until you commit it. + /// fd.commit()?; + /// Ok(()) + /// } + /// ``` + pub fn create(cache: P, key: K) -> Result + where + P: AsRef, + K: AsRef, + { + WriteOpts::new() + .algorithm(Algorithm::Sha256) + .open_sync(cache.as_ref(), key.as_ref()) + .with_context(|| { + format!( + "Failed to open a write handle for key {} for cache at {:?}", + key.as_ref(), + cache.as_ref() + ) + }) + } + + /// Closes the Writer handle and writes content and index entries. Also /// verifies data against `size` and `integrity` options, if provided. /// Must be called manually in order to complete the writing process, /// otherwise everything will be thrown out. @@ -343,16 +391,14 @@ impl SyncPut { #[cfg(test)] mod tests { - use async_std::task; + use async_attributes; - #[test] - fn round_trip() { + #[async_attributes::test] + async fn round_trip() { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - task::block_on(async { - crate::put::data(&dir, "hello", b"hello").await.unwrap(); - }); - let data = task::block_on(async { crate::get::data(&dir, "hello").await.unwrap() }); + crate::write(&dir, "hello", b"hello").await.unwrap(); + let data = crate::read(&dir, "hello").await.unwrap(); assert_eq!(data, b"hello"); } @@ -360,8 +406,8 @@ mod tests { fn round_trip_sync() { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - crate::put::data_sync(&dir, "hello", b"hello").unwrap(); - let data = crate::get::data_sync(&dir, "hello").unwrap(); + crate::write_sync(&dir, "hello", b"hello").unwrap(); + let data = crate::read_sync(&dir, "hello").unwrap(); assert_eq!(data, b"hello"); } } diff --git a/src/rm.rs b/src/rm.rs index b9b2c00..87026c0 100644 --- a/src/rm.rs +++ b/src/rm.rs @@ -10,8 +10,8 @@ use ssri::Integrity; use crate::content::rm; use crate::index; -/// Removes an individual index entry. The associated content will be left -/// intact. +/// Removes an individual index metadata entry. The associated content will be +/// left in the cache. /// /// ## Example /// ```no_run @@ -21,19 +21,20 @@ use crate::index; /// /// #[async_attributes::main] /// async fn main() -> Result<()> { -/// let sri = cacache::put::data("./my-cache", "my-key", b"hello").await?; +/// let sri = cacache::write("./my-cache", "my-key", b"hello").await?; /// -/// cacache::rm::entry("./my-cache", "my-key").await?; +/// cacache::remove("./my-cache", "my-key").await?; /// /// // This fails: -/// cacache::get::data("./my-cache", "my-key").await?; +/// cacache::read("./my-cache", "my-key").await?; /// /// // But this succeeds: -/// cacache::get::data_hash("./my-cache", &sri).await?; +/// cacache::read_hash("./my-cache", &sri).await?; +/// /// Ok(()) /// } /// ``` -pub async fn entry(cache: P, key: K) -> Result<()> +pub async fn remove(cache: P, key: K) -> Result<()> where P: AsRef, K: AsRef, @@ -60,21 +61,21 @@ where /// /// #[async_attributes::main] /// async fn main() -> Result<()> { -/// let sri = cacache::put::data("./my-cache", "my-key", b"hello").await?; +/// let sri = cacache::write("./my-cache", "my-key", b"hello").await?; /// -/// cacache::rm::entry("./my-cache", "my-key").await?; +/// cacache::remove_hash("./my-cache", &sri).await?; /// /// // These fail: -/// cacache::get::data("./my-cache", "my-key").await?; -/// cacache::get::data_hash("./my-cache", &sri).await?; +/// cacache::read("./my-cache", "my-key").await?; +/// cacache::read_hash("./my-cache", &sri).await?; /// /// // But this succeeds: -/// cacache::get::entry("./my-cache", "my-key").await?; +/// cacache::metadata("./my-cache", "my-key").await?; /// /// Ok(()) /// } /// ``` -pub async fn content>(cache: P, sri: &Integrity) -> Result<()> { +pub async fn remove_hash>(cache: P, sri: &Integrity) -> Result<()> { rm::rm_async(cache.as_ref(), &sri).await.with_context(|| { format!( "Failed to remove content under {} in cache at {:?}", @@ -95,19 +96,19 @@ pub async fn content>(cache: P, sri: &Integrity) -> Result<()> { /// /// #[async_attributes::main] /// async fn main() -> Result<()> { -/// let sri = cacache::put::data("./my-cache", "my-key", b"hello").await?; +/// let sri = cacache::write("./my-cache", "my-key", b"hello").await?; /// -/// cacache::rm::entry("./my-cache", "my-key").await?; +/// cacache::clear("./my-cache").await?; /// /// // These all fail: -/// cacache::get::data("./my-cache", "my-key").await?; -/// cacache::get::entry("./my-cache", "my-key").await?; -/// cacache::get::data_hash("./my-cache", &sri).await?; +/// cacache::read("./my-cache", "my-key").await?; +/// cacache::metadata("./my-cache", "my-key").await?; +/// cacache::read_hash("./my-cache", &sri).await?; /// /// Ok(()) /// } /// ``` -pub async fn all>(cache: P) -> Result<()> { +pub async fn clear>(cache: P) -> Result<()> { for entry in cache.as_ref().read_dir()? { if let Ok(entry) = entry { afs::remove_dir_all(entry.path()).await?; @@ -117,7 +118,7 @@ pub async fn all>(cache: P) -> Result<()> { } /// Removes an individual index entry synchronously. The associated content -/// will be left intact. +/// will be left in the cache. /// /// ## Example /// ```no_run @@ -125,20 +126,20 @@ pub async fn all>(cache: P) -> Result<()> { /// use std::io::Read; /// /// fn main() -> Result<()> { -/// let sri = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; +/// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?; /// -/// cacache::rm::entry_sync("./my-cache", "my-key")?; +/// cacache::remove_sync("./my-cache", "my-key")?; /// /// // This fails: -/// cacache::get::data_sync("./my-cache", "my-key")?; +/// cacache::read_sync("./my-cache", "my-key")?; /// /// // But this succeeds: -/// cacache::get::data_hash_sync("./my-cache", &sri)?; +/// cacache::read_hash_sync("./my-cache", &sri)?; /// /// Ok(()) /// } /// ``` -pub fn entry_sync(cache: P, key: K) -> Result<()> +pub fn remove_sync(cache: P, key: K) -> Result<()> where P: AsRef, K: AsRef, @@ -161,21 +162,21 @@ where /// use std::io::Read; /// /// fn main() -> Result<()> { -/// let sri = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; +/// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?; /// -/// cacache::rm::entry_sync("./my-cache", "my-key")?; +/// cacache::remove_hash_sync("./my-cache", &sri)?; /// /// // These fail: -/// cacache::get::data_sync("./my-cache", "my-key")?; -/// cacache::get::data_hash_sync("./my-cache", &sri)?; +/// cacache::read_sync("./my-cache", "my-key")?; +/// cacache::read_hash_sync("./my-cache", &sri)?; /// /// // But this succeeds: -/// cacache::get::entry_sync("./my-cache", "my-key")?; +/// cacache::metadata_sync("./my-cache", "my-key")?; /// /// Ok(()) /// } /// ``` -pub fn content_sync>(cache: P, sri: &Integrity) -> Result<()> { +pub fn remove_hash_sync>(cache: P, sri: &Integrity) -> Result<()> { rm::rm(cache.as_ref(), &sri).with_context(|| { format!( "Failed to remove content under {} in cache at {:?}", @@ -194,19 +195,19 @@ pub fn content_sync>(cache: P, sri: &Integrity) -> Result<()> { /// use std::io::Read; /// /// fn main() -> Result<()> { -/// let sri = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; +/// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?; /// -/// cacache::rm::entry_sync("./my-cache", "my-key")?; +/// cacache::clear_sync("./my-cache")?; /// /// // These all fail: -/// cacache::get::data_sync("./my-cache", "my-key")?; -/// cacache::get::data_hash_sync("./my-cache", &sri)?; -/// cacache::get::entry_sync("./my-cache", "my-key")?; +/// cacache::read_sync("./my-cache", "my-key")?; +/// cacache::read_hash_sync("./my-cache", &sri)?; +/// cacache::metadata_sync("./my-cache", "my-key")?; /// /// Ok(()) /// } /// ``` -pub fn all_sync>(cache: P) -> Result<()> { +pub fn clear_sync>(cache: P) -> Result<()> { for entry in cache.as_ref().read_dir()? { if let Ok(entry) = entry { fs::remove_dir_all(entry.path())?; @@ -220,98 +221,98 @@ mod tests { use async_std::task; #[test] - fn entry() { + fn test_remove() { task::block_on(async { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - let sri = crate::put::data(&dir, "key", b"my-data").await.unwrap(); + let sri = crate::write(&dir, "key", b"my-data").await.unwrap(); - crate::rm::entry(&dir, "key").await.unwrap(); + crate::remove(&dir, "key").await.unwrap(); - let entry = crate::get::entry(&dir, "key").await.unwrap(); + let entry = crate::metadata(&dir, "key").await.unwrap(); assert_eq!(entry, None); - let data_exists = crate::get::hash_exists(&dir, &sri).await; + let data_exists = crate::exists(&dir, &sri).await; assert_eq!(data_exists, true); }); } #[test] - fn content() { + fn test_remove_data() { task::block_on(async { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - let sri = crate::put::data(&dir, "key", b"my-data").await.unwrap(); + let sri = crate::write(&dir, "key", b"my-data").await.unwrap(); - crate::rm::content(&dir, &sri).await.unwrap(); + crate::remove_hash(&dir, &sri).await.unwrap(); - let entry = crate::get::entry(&dir, "key").await.unwrap(); + let entry = crate::metadata(&dir, "key").await.unwrap(); assert_eq!(entry.is_some(), true); - let data_exists = crate::get::hash_exists(&dir, &sri).await; + let data_exists = crate::exists(&dir, &sri).await; assert_eq!(data_exists, false); }); } #[test] - fn all() { + fn test_clear() { task::block_on(async { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - let sri = crate::put::data(&dir, "key", b"my-data").await.unwrap(); + let sri = crate::write(&dir, "key", b"my-data").await.unwrap(); - crate::rm::all(&dir).await.unwrap(); + crate::clear(&dir).await.unwrap(); - let entry = crate::get::entry(&dir, "key").await.unwrap(); + let entry = crate::metadata(&dir, "key").await.unwrap(); assert_eq!(entry.is_some(), false); - let data_exists = crate::get::hash_exists(&dir, &sri).await; + let data_exists = crate::exists(&dir, &sri).await; assert_eq!(data_exists, false); }); } #[test] - fn entry_sync() { + fn test_remove_sync() { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - let sri = crate::put::data_sync(&dir, "key", b"my-data").unwrap(); + let sri = crate::write_sync(&dir, "key", b"my-data").unwrap(); - crate::rm::entry_sync(&dir, "key").unwrap(); + crate::remove_sync(&dir, "key").unwrap(); - let new_entry = crate::get::entry_sync(&dir, "key").unwrap(); + let new_entry = crate::metadata_sync(&dir, "key").unwrap(); assert_eq!(new_entry, None); - let data_exists = crate::get::hash_exists_sync(&dir, &sri); + let data_exists = crate::exists_sync(&dir, &sri); assert_eq!(data_exists, true); } #[test] - fn content_sync() { + fn test_remove_data_sync() { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - let sri = crate::put::data_sync(&dir, "key", b"my-data").unwrap(); + let sri = crate::write_sync(&dir, "key", b"my-data").unwrap(); - crate::rm::content_sync(&dir, &sri).unwrap(); + crate::remove_hash_sync(&dir, &sri).unwrap(); - let new_entry = crate::get::entry_sync(&dir, "key").unwrap(); - assert_eq!(new_entry.is_some(), true); + let entry = crate::metadata_sync(&dir, "key").unwrap(); + assert_eq!(entry.is_some(), true); - let data_exists = crate::get::hash_exists_sync(&dir, &sri); + let data_exists = crate::exists_sync(&dir, &sri); assert_eq!(data_exists, false); } #[test] - fn all_sync() { + fn test_clear_sync() { let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned(); - let sri = crate::put::data_sync(&dir, "key", b"my-data").unwrap(); + let sri = crate::write_sync(&dir, "key", b"my-data").unwrap(); - crate::rm::all_sync(&dir).unwrap(); + crate::clear_sync(&dir).unwrap(); - let new_entry = crate::get::entry_sync(&dir, "key").unwrap(); - assert_eq!(new_entry, None); + let entry = crate::metadata_sync(&dir, "key").unwrap(); + assert_eq!(entry, None); - let data_exists = crate::get::hash_exists_sync(&dir, &sri); + let data_exists = crate::exists_sync(&dir, &sri); assert_eq!(data_exists, false); } }