feat(api): rewrite entire API to be like std::fs (#21)

Fixes: #13

BREAKING CHANGE: The entire API surface has been rewritten and overhauled. Instead of breaking up the package into get/put/rm/ls modules, everything is now available mostly through the toplevel module, following the same conventions as std:fs for its operations. Associated structs and other types have also been renamed. Please refer to the documentation for the new API, and I hope you find it more ergonomic!
This commit is contained in:
Kat Marchán 2019-10-23 21:17:57 -07:00 committed by GitHub
parent 0395b0fbff
commit 743476b274
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 686 additions and 476 deletions

View File

@ -13,10 +13,10 @@ async fn main() -> Result<(), cacache::Error> {
let dir = String::from("./my-cache"); let dir = String::from("./my-cache");
// Write some data! // Write some data!
cacache::put::data(&dir, "key", b"my-async-data").await?; cacache::write(&dir, "key", b"my-async-data").await?;
// Get the data back! // Get the data back!
let data = cacache::get::data(&dir, "key").await?; let data = cacache::read(&dir, "key").await?;
assert_eq!(data, b"my-async-data"); assert_eq!(data, b"my-async-data");
// Clean up the data! // Clean up the data!
@ -36,11 +36,13 @@ Using [`cargo-edit`](https://crates.io/crates/cargo-edit)
## Features ## Features
- First-class async support, using [`async-std`](https://crates.io/crates/async-std) as its runtime. Sync APIs are available but secondary. - First-class async support, using [`async-std`](https://crates.io/crates/async-std) as its runtime. Sync APIs are available but secondary
- `std::fs`-style API
- Extraction by key or by content address (shasum, etc) - Extraction by key or by content address (shasum, etc)
- [Subresource Integrity](#integrity) web standard support - [Subresource Integrity](#integrity) web standard support
- Multi-hash support - safely host sha1, sha512, etc, in a single cache - Multi-hash support - safely host sha1, sha512, etc, in a single cache
- Automatic content deduplication - Automatic content deduplication
- Atomic content writes even for large data
- Fault tolerance (immune to corruption, partial writes, process races, etc) - Fault tolerance (immune to corruption, partial writes, process races, etc)
- Consistency guarantees on read and write (full data verification) - Consistency guarantees on read and write (full data verification)
- Lockless, high-concurrency cache access - Lockless, high-concurrency cache access
@ -48,6 +50,7 @@ Using [`cargo-edit`](https://crates.io/crates/cargo-edit)
- Large file support - Large file support
- Pretty darn fast - Pretty darn fast
- Arbitrary metadata storage - Arbitrary metadata storage
- Cross-platform: Windows and case-(in)sensitive filesystem support
- Punches nazis - Punches nazis
## Contributing ## Contributing

View File

@ -73,17 +73,17 @@ fn baseline_read_many_async(c: &mut Criterion) {
}); });
} }
fn get_data_hash_sync(c: &mut Criterion) { fn read_hash_sync(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned(); let cache = tmp.path().to_owned();
let data = b"hello world".to_vec(); let data = b"hello world".to_vec();
let sri = cacache::put::data_sync(&cache, "hello", data).unwrap(); let sri = cacache::write_sync(&cache, "hello", data).unwrap();
c.bench_function("get::data_hash_sync", move |b| { c.bench_function("get::data_hash_sync", move |b| {
b.iter(|| cacache::get::data_hash_sync(black_box(&cache), black_box(&sri)).unwrap()) b.iter(|| cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap())
}); });
} }
fn get_data_hash_many_sync(c: &mut Criterion) { fn read_hash_many_sync(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned(); let cache = tmp.path().to_owned();
let data: Vec<_> = (0..) let data: Vec<_> = (0..)
@ -92,41 +92,38 @@ fn get_data_hash_many_sync(c: &mut Criterion) {
.collect(); .collect();
let sris: Vec<_> = data let sris: Vec<_> = data
.iter() .iter()
.map(|datum| cacache::put::data_sync(&cache, "hello", datum).unwrap()) .map(|datum| cacache::write_sync(&cache, "hello", datum).unwrap())
.collect(); .collect();
c.bench_function("get::data_hash_many_sync", move |b| { c.bench_function("get::data_hash_many_sync", move |b| {
b.iter(|| { b.iter(|| {
for sri in sris.iter() { for sri in sris.iter() {
cacache::get::data_hash_sync(black_box(&cache), black_box(&sri)).unwrap(); cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap();
} }
}) })
}); });
} }
fn get_data_sync(c: &mut Criterion) { fn read_sync(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned(); let cache = tmp.path().to_owned();
let data = b"hello world".to_vec(); let data = b"hello world".to_vec();
cacache::put::data_sync(&cache, "hello", data).unwrap(); cacache::write_sync(&cache, "hello", data).unwrap();
cacache::get::data_sync(&cache, "hello").unwrap();
c.bench_function("get::data_sync", move |b| { c.bench_function("get::data_sync", move |b| {
b.iter(|| { b.iter(|| cacache::read_sync(black_box(&cache), black_box(String::from("hello"))).unwrap())
cacache::get::data_sync(black_box(&cache), black_box(String::from("hello"))).unwrap()
})
}); });
} }
fn get_data_hash_sync_big_data(c: &mut Criterion) { fn read_hash_sync_big_data(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned(); let cache = tmp.path().to_owned();
let data = vec![1; 1024 * 1024 * 5]; let data = vec![1; 1024 * 1024 * 5];
let sri = cacache::put::data_sync(&cache, "hello", data).unwrap(); let sri = cacache::write_sync(&cache, "hello", data).unwrap();
c.bench_function("get_hash_big_data", move |b| { c.bench_function("get_hash_big_data", move |b| {
b.iter(|| cacache::get::data_hash_sync(black_box(&cache), black_box(&sri)).unwrap()) b.iter(|| cacache::read_hash_sync(black_box(&cache), black_box(&sri)).unwrap())
}); });
} }
fn get_data_hash_many_async(c: &mut Criterion) { fn read_hash_many_async(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned(); let cache = tmp.path().to_owned();
let data: Vec<_> = (0..) let data: Vec<_> = (0..)
@ -135,51 +132,45 @@ fn get_data_hash_many_async(c: &mut Criterion) {
.collect(); .collect();
let sris: Vec<_> = data let sris: Vec<_> = data
.iter() .iter()
.map(|datum| cacache::put::data_sync(&cache, "hello", datum).unwrap()) .map(|datum| cacache::write_sync(&cache, "hello", datum).unwrap())
.collect(); .collect();
c.bench_function("get::data_hash_many", move |b| { c.bench_function("get::data_hash_many", move |b| {
b.iter(|| { b.iter(|| {
let tasks = sris let tasks = sris
.iter() .iter()
.map(|sri| cacache::get::data_hash(black_box(&cache), black_box(&sri))); .map(|sri| cacache::read_hash(black_box(&cache), black_box(&sri)));
task::block_on(futures::future::join_all(tasks)); task::block_on(futures::future::join_all(tasks));
}) })
}); });
} }
fn get_data_hash_async(c: &mut Criterion) { fn read_hash_async(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned(); let cache = tmp.path().to_owned();
let data = b"hello world".to_vec(); let data = b"hello world".to_vec();
let sri = cacache::put::data_sync(&cache, "hello", data).unwrap(); let sri = cacache::write_sync(&cache, "hello", data).unwrap();
c.bench_function("get::data_hash", move |b| { c.bench_function("get::data_hash", move |b| {
b.iter(|| { b.iter(|| task::block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap())
task::block_on(cacache::get::data_hash(black_box(&cache), black_box(&sri))).unwrap()
})
}); });
} }
fn get_data_async(c: &mut Criterion) { fn read_async(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned(); let cache = tmp.path().to_owned();
let data = b"hello world".to_vec(); let data = b"hello world".to_vec();
cacache::put::data_sync(&cache, "hello", data).unwrap(); cacache::write_sync(&cache, "hello", data).unwrap();
c.bench_function("get::data", move |b| { c.bench_function("get::data", move |b| {
b.iter(|| { b.iter(|| task::block_on(cacache::read(black_box(&cache), black_box("hello"))).unwrap())
task::block_on(cacache::get::data(black_box(&cache), black_box("hello"))).unwrap()
})
}); });
} }
fn get_data_hash_async_big_data(c: &mut Criterion) { fn read_hash_async_big_data(c: &mut Criterion) {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path().to_owned(); let cache = tmp.path().to_owned();
let data = vec![1; 1024 * 1024 * 5]; let data = vec![1; 1024 * 1024 * 5];
let sri = cacache::put::data_sync(&cache, "hello", data).unwrap(); let sri = cacache::write_sync(&cache, "hello", data).unwrap();
c.bench_function("get::data_big_data", move |b| { c.bench_function("get::data_big_data", move |b| {
b.iter(|| { b.iter(|| task::block_on(cacache::read_hash(black_box(&cache), black_box(&sri))).unwrap())
task::block_on(cacache::get::data_hash(black_box(&cache), black_box(&sri))).unwrap()
})
}); });
} }
@ -189,13 +180,13 @@ criterion_group!(
baseline_read_many_sync, baseline_read_many_sync,
baseline_read_async, baseline_read_async,
baseline_read_many_async, baseline_read_many_async,
get_data_hash_async, read_hash_async,
get_data_hash_many_async, read_hash_many_async,
get_data_hash_sync, read_hash_sync,
get_data_hash_many_sync, read_hash_many_sync,
get_data_async, read_async,
get_data_sync, read_sync,
get_data_hash_async_big_data, read_hash_async_big_data,
get_data_hash_sync_big_data read_hash_sync_big_data
); );
criterion_main!(benches); criterion_main!(benches);

View File

@ -8,23 +8,23 @@ use futures::prelude::*;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use ssri::{Algorithm, Integrity}; use ssri::{Algorithm, Integrity};
use crate::content::read::{self, AsyncReader, Reader}; use crate::content::read;
use crate::errors::Error; use crate::errors::Error;
use crate::index::{self, Entry}; use crate::index::{self, Metadata};
// --------- // ---------
// Async API // Async API
// --------- // ---------
/// File handle for asynchronously reading from a content entry. /// File handle for reading data asynchronously.
/// ///
/// Make sure to call `.check()` when done reading to verify that the /// Make sure to call `.check()` when done reading to verify that the
/// extracted data passes integrity verification. /// extracted data passes integrity verification.
pub struct AsyncGet { pub struct Reader {
reader: AsyncReader, reader: read::AsyncReader,
} }
impl AsyncRead for AsyncGet { impl AsyncRead for Reader {
fn poll_read( fn poll_read(
mut self: Pin<&mut Self>, mut self: Pin<&mut Self>,
cx: &mut TaskContext<'_>, cx: &mut TaskContext<'_>,
@ -34,11 +34,15 @@ impl AsyncRead for AsyncGet {
} }
} }
impl AsyncGet { impl Reader {
/// Checks that data read from disk passes integrity checks. Returns the /// Checks that data read from disk passes integrity checks. Returns the
/// algorithm that was used verified the data. Should be called only after /// algorithm that was used verified the data. Should be called only after
/// all data has been read from disk. /// all data has been read from disk.
/// ///
/// This check is very cheap, since most of the verification is done on
/// the fly. This simply finalizes verification, and is always
/// synchronous.
///
/// ## Example /// ## Example
/// ```no_run /// ```no_run
/// use async_std::prelude::*; /// use async_std::prelude::*;
@ -47,11 +51,11 @@ impl AsyncGet {
/// ///
/// #[async_attributes::main] /// #[async_attributes::main]
/// async fn main() -> Result<()> { /// async fn main() -> Result<()> {
/// let mut handle = cacache::get::open("./my-cache", "my-key").await?; /// let mut fd = cacache::Reader::open("./my-cache", "my-key").await?;
/// let mut str = String::new(); /// let mut str = String::new();
/// handle.read_to_string(&mut str).await?; /// fd.read_to_string(&mut str).await?;
/// // Remember to check that the data you got was correct! /// // Remember to check that the data you got was correct!
/// handle.check()?; /// fd.check()?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
@ -60,68 +64,67 @@ impl AsyncGet {
.check() .check()
.context("Cache read data verification check failed.") .context("Cache read data verification check failed.")
} }
} /// Opens a new file handle into the cache, looking it up in the index using
/// `key`.
/// Opens a new file handle into the cache, looking it up in the index using ///
/// `key`. /// ## Example
/// /// ```no_run
/// ## Example /// use async_std::prelude::*;
/// ```no_run /// use async_attributes;
/// use async_std::prelude::*; /// use anyhow::Result;
/// use async_attributes; ///
/// use anyhow::Result; /// #[async_attributes::main]
/// /// async fn main() -> Result<()> {
/// #[async_attributes::main] /// let mut fd = cacache::Reader::open("./my-cache", "my-key").await?;
/// async fn main() -> Result<()> { /// let mut str = String::new();
/// let mut handle = cacache::get::open("./my-cache", "my-key").await?; /// fd.read_to_string(&mut str).await?;
/// let mut str = String::new(); /// // Remember to check that the data you got was correct!
/// handle.read_to_string(&mut str).await?; /// fd.check()?;
/// // Remember to check that the data you got was correct! /// Ok(())
/// handle.check()?; /// }
/// Ok(()) /// ```
/// } pub async fn open<P, K>(cache: P, key: K) -> Result<Reader>
/// ``` where
pub async fn open<P, K>(cache: P, key: K) -> Result<AsyncGet> P: AsRef<Path>,
where K: AsRef<str>,
P: AsRef<Path>, {
K: AsRef<str>, if let Some(entry) = index::find_async(cache.as_ref(), key.as_ref()).await? {
{ Reader::open_hash(cache, entry.integrity).await
if let Some(entry) = index::find_async(cache.as_ref(), key.as_ref()).await? { } else {
open_hash(cache, entry.integrity).await Err(Error::EntryNotFound(
} else { cache.as_ref().to_path_buf(),
Err(Error::EntryNotFound( key.as_ref().into(),
cache.as_ref().to_path_buf(), ))?
key.as_ref().into(), }
))?
} }
}
/// Opens a new file handle into the cache, based on its integrity address. /// Opens a new file handle into the cache, based on its integrity address.
/// ///
/// ## Example /// ## Example
/// ```no_run /// ```no_run
/// use async_std::prelude::*; /// use async_std::prelude::*;
/// use async_attributes; /// use async_attributes;
/// use anyhow::Result; /// use anyhow::Result;
/// ///
/// #[async_attributes::main] /// #[async_attributes::main]
/// async fn main() -> Result<()> { /// async fn main() -> Result<()> {
/// let sri = cacache::put::data("./my-cache", "key", b"hello world").await?; /// let sri = cacache::write("./my-cache", "key", b"hello world").await?;
/// let mut handle = cacache::get::open_hash("./my-cache", sri).await?; /// let mut fd = cacache::Reader::open_hash("./my-cache", sri).await?;
/// let mut str = String::new(); /// let mut str = String::new();
/// handle.read_to_string(&mut str).await?; /// fd.read_to_string(&mut str).await?;
/// // Remember to check that the data you got was correct! /// // Remember to check that the data you got was correct!
/// handle.check()?; /// fd.check()?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub async fn open_hash<P>(cache: P, sri: Integrity) -> Result<AsyncGet> pub async fn open_hash<P>(cache: P, sri: Integrity) -> Result<Reader>
where where
P: AsRef<Path>, P: AsRef<Path>,
{ {
Ok(AsyncGet { Ok(Reader {
reader: read::open_async(cache.as_ref(), sri).await?, reader: read::open_async(cache.as_ref(), sri).await?,
}) })
}
} }
/// Reads the entire contents of a cache file into a bytes vector, looking the /// Reads the entire contents of a cache file into a bytes vector, looking the
@ -135,17 +138,47 @@ where
/// ///
/// #[async_attributes::main] /// #[async_attributes::main]
/// async fn main() -> Result<()> { /// async fn main() -> Result<()> {
/// let data = cacache::get::data("./my-cache", "my-key").await?; /// let data: Vec<u8> = cacache::read("./my-cache", "my-key").await?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub async fn data<P, K>(cache: P, key: K) -> Result<Vec<u8>> pub async fn read<P, K>(cache: P, key: K) -> Result<Vec<u8>>
where where
P: AsRef<Path>, P: AsRef<Path>,
K: AsRef<str>, K: AsRef<str>,
{ {
if let Some(entry) = index::find_async(cache.as_ref(), key.as_ref()).await? { if let Some(entry) = index::find_async(cache.as_ref(), key.as_ref()).await? {
data_hash(cache, &entry.integrity).await read_hash(cache, &entry.integrity).await
} else {
Err(Error::EntryNotFound(
cache.as_ref().to_path_buf(),
key.as_ref().into(),
))?
}
}
/// Reads the entire contents of a cache file into a string, looking the
/// data up by key.
///
/// ## Example
/// ```no_run
/// use async_std::prelude::*;
/// use async_attributes;
/// use anyhow::Result;
///
/// #[async_attributes::main]
/// async fn main() -> Result<()> {
/// let str: String = cacache::read_to_string("./my-cache", "my-key").await?;
/// Ok(())
/// }
/// ```
pub async fn read_to_string<P, K>(cache: P, key: K) -> Result<String>
where
P: AsRef<Path>,
K: AsRef<str>,
{
if let Some(entry) = index::find_async(cache.as_ref(), key.as_ref()).await? {
read_hash_to_string(cache, &entry.integrity).await
} else { } else {
Err(Error::EntryNotFound( Err(Error::EntryNotFound(
cache.as_ref().to_path_buf(), cache.as_ref().to_path_buf(),
@ -165,19 +198,20 @@ where
/// ///
/// #[async_attributes::main] /// #[async_attributes::main]
/// async fn main() -> Result<()> { /// async fn main() -> Result<()> {
/// let sri = cacache::put::data("./my-cache", "my-key", b"hello").await?; /// let sri = cacache::write("./my-cache", "my-key", b"hello").await?;
/// let data = cacache::get::data_hash("./my-cache", &sri).await?; /// let data: Vec<u8> = cacache::read_hash("./my-cache", &sri).await?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub async fn data_hash<P>(cache: P, sri: &Integrity) -> Result<Vec<u8>> pub async fn read_hash<P>(cache: P, sri: &Integrity) -> Result<Vec<u8>>
where where
P: AsRef<Path>, P: AsRef<Path>,
{ {
Ok(read::read_async(cache.as_ref(), sri).await?) Ok(read::read_async(cache.as_ref(), sri).await?)
} }
/// Copies a cache entry by key to a specified location. /// Reads the entire contents of a cache file into a string, looking the
/// data up by its content address.
/// ///
/// ## Example /// ## Example
/// ```no_run /// ```no_run
@ -187,7 +221,32 @@ where
/// ///
/// #[async_attributes::main] /// #[async_attributes::main]
/// async fn main() -> Result<()> { /// async fn main() -> Result<()> {
/// cacache::get::copy("./my-cache", "my-key", "./data.txt").await?; /// let sri = cacache::write("./my-cache", "my-key", b"hello").await?;
/// let str: String = cacache::read_hash_to_string("./my-cache", &sri).await?;
/// Ok(())
/// }
/// ```
pub async fn read_hash_to_string<P>(cache: P, sri: &Integrity) -> Result<String>
where
P: AsRef<Path>,
{
Ok(String::from_utf8(
read::read_async(cache.as_ref(), sri).await?,
)?)
}
/// Copies cache data to a specified location. Returns the number of bytes
/// copied.
///
/// ## Example
/// ```no_run
/// use async_std::prelude::*;
/// use async_attributes;
/// use anyhow::Result;
///
/// #[async_attributes::main]
/// async fn main() -> Result<()> {
/// cacache::copy("./my-cache", "my-key", "./data.txt").await?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
@ -207,7 +266,8 @@ where
} }
} }
/// Copies a cache entry by integrity address to a specified location. /// Copies a cache data by hash to a specified location. Returns the number of
/// bytes copied.
/// ///
/// ## Example /// ## Example
/// ```no_run /// ```no_run
@ -217,8 +277,8 @@ where
/// ///
/// #[async_attributes::main] /// #[async_attributes::main]
/// async fn main() -> Result<()> { /// async fn main() -> Result<()> {
/// let sri = cacache::put::data("./my-cache", "my-key", b"hello world").await?; /// let sri = cacache::write("./my-cache", "my-key", b"hello world").await?;
/// cacache::get::copy_hash("./my-cache", &sri, "./data.txt").await?; /// cacache::copy_hash("./my-cache", &sri, "./data.txt").await?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
@ -230,8 +290,12 @@ where
read::copy_async(cache.as_ref(), sri, to.as_ref()).await read::copy_async(cache.as_ref(), sri, to.as_ref()).await
} }
/// Gets entry information and metadata for a certain key. /// Gets the metadata entry for a certain key.
pub async fn entry<P, K>(cache: P, key: K) -> Result<Option<Entry>> ///
/// Note that the existence of a metadata entry is not a guarantee that the
/// underlying data exists, since they are stored and managed independently.
/// To verify that the underlying associated data exists, use `exists()`.
pub async fn metadata<P, K>(cache: P, key: K) -> Result<Option<Metadata>>
where where
P: AsRef<Path>, P: AsRef<Path>,
K: AsRef<str>, K: AsRef<str>,
@ -240,7 +304,7 @@ where
} }
/// Returns true if the given hash exists in the cache. /// Returns true if the given hash exists in the cache.
pub async fn hash_exists<P: AsRef<Path>>(cache: P, sri: &Integrity) -> bool { pub async fn exists<P: AsRef<Path>>(cache: P, sri: &Integrity) -> bool {
read::has_content_async(cache.as_ref(), &sri) read::has_content_async(cache.as_ref(), &sri)
.await .await
.is_some() .is_some()
@ -250,22 +314,22 @@ pub async fn hash_exists<P: AsRef<Path>>(cache: P, sri: &Integrity) -> bool {
// Synchronous API // Synchronous API
// --------------- // ---------------
/// File handle for reading from a content entry. /// File handle for reading data synchronously.
/// ///
/// Make sure to call `get.check()` when done reading /// Make sure to call `get.check()` when done reading
/// to verify that the extracted data passes integrity /// to verify that the extracted data passes integrity
/// verification. /// verification.
pub struct SyncGet { pub struct SyncReader {
reader: Reader, reader: read::Reader,
} }
impl std::io::Read for SyncGet { impl std::io::Read for SyncReader {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> { fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self.reader.read(buf) self.reader.read(buf)
} }
} }
impl SyncGet { impl SyncReader {
/// Checks that data read from disk passes integrity checks. Returns the /// Checks that data read from disk passes integrity checks. Returns the
/// algorithm that was used verified the data. Should be called only after /// algorithm that was used verified the data. Should be called only after
/// all data has been read from disk. /// all data has been read from disk.
@ -276,11 +340,11 @@ impl SyncGet {
/// use std::io::Read; /// use std::io::Read;
/// ///
/// fn main() -> Result<()> { /// fn main() -> Result<()> {
/// let mut handle = cacache::get::open_sync("./my-cache", "my-key")?; /// let mut fd = cacache::SyncReader::open("./my-cache", "my-key")?;
/// let mut str = String::new(); /// let mut str = String::new();
/// handle.read_to_string(&mut str)?; /// fd.read_to_string(&mut str)?;
/// // Remember to check that the data you got was correct! /// // Remember to check that the data you got was correct!
/// handle.check()?; /// fd.check()?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
@ -289,64 +353,64 @@ impl SyncGet {
.check() .check()
.context("Cache read data verification check failed.") .context("Cache read data verification check failed.")
} }
}
/// Opens a new synchronous file handle into the cache, looking it up in the /// Opens a new synchronous file handle into the cache, looking it up in the
/// index using `key`. /// index using `key`.
/// ///
/// ## Example /// ## Example
/// ```no_run /// ```no_run
/// use anyhow::Result; /// use anyhow::Result;
/// use std::io::Read; /// use std::io::Read;
/// ///
/// fn main() -> Result<()> { /// fn main() -> Result<()> {
/// let mut handle = cacache::get::open_sync("./my-cache", "my-key")?; /// let mut fd = cacache::SyncReader::open("./my-cache", "my-key")?;
/// let mut str = String::new(); /// let mut str = String::new();
/// handle.read_to_string(&mut str)?; /// fd.read_to_string(&mut str)?;
/// // Remember to check that the data you got was correct! /// // Remember to check that the data you got was correct!
/// handle.check()?; /// fd.check()?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub fn open_sync<P, K>(cache: P, key: K) -> Result<SyncGet> pub fn open<P, K>(cache: P, key: K) -> Result<SyncReader>
where where
P: AsRef<Path>, P: AsRef<Path>,
K: AsRef<str>, K: AsRef<str>,
{ {
if let Some(entry) = index::find(cache.as_ref(), key.as_ref())? { if let Some(entry) = index::find(cache.as_ref(), key.as_ref())? {
open_hash_sync(cache, entry.integrity) SyncReader::open_hash(cache, entry.integrity)
} else { } else {
Err(Error::EntryNotFound( Err(Error::EntryNotFound(
cache.as_ref().to_path_buf(), cache.as_ref().to_path_buf(),
key.as_ref().into(), key.as_ref().into(),
))? ))?
}
} }
}
/// Opens a new synchronous file handle into the cache, based on its integrity address. /// Opens a new synchronous file handle into the cache, based on its integrity address.
/// ///
/// ## Example /// ## Example
/// ```no_run /// ```no_run
/// use anyhow::Result; /// use anyhow::Result;
/// use std::io::Read; /// use std::io::Read;
/// ///
/// fn main() -> Result<()> { /// fn main() -> Result<()> {
/// let sri = cacache::put::data_sync("./my-cache", "key", b"hello world")?; /// let sri = cacache::write_sync("./my-cache", "key", b"hello world")?;
/// let mut handle = cacache::get::open_hash_sync("./my-cache", sri)?; /// let mut fd = cacache::SyncReader::open_hash("./my-cache", sri)?;
/// let mut str = String::new(); /// let mut str = String::new();
/// handle.read_to_string(&mut str)?; /// fd.read_to_string(&mut str)?;
/// // Remember to check that the data you got was correct! /// // Remember to check that the data you got was correct!
/// handle.check()?; /// fd.check()?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub fn open_hash_sync<P>(cache: P, sri: Integrity) -> Result<SyncGet> pub fn open_hash<P>(cache: P, sri: Integrity) -> Result<SyncReader>
where where
P: AsRef<Path>, P: AsRef<Path>,
{ {
Ok(SyncGet { Ok(SyncReader {
reader: read::open(cache.as_ref(), sri)?, reader: read::open(cache.as_ref(), sri)?,
}) })
}
} }
/// Reads the entire contents of a cache file synchronously into a bytes /// Reads the entire contents of a cache file synchronously into a bytes
@ -358,17 +422,17 @@ where
/// use std::io::Read; /// use std::io::Read;
/// ///
/// fn main() -> Result<()> { /// fn main() -> Result<()> {
/// let data = cacache::get::data_sync("./my-cache", "my-key")?; /// let data = cacache::read_sync("./my-cache", "my-key")?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub fn data_sync<P, K>(cache: P, key: K) -> Result<Vec<u8>> pub fn read_sync<P, K>(cache: P, key: K) -> Result<Vec<u8>>
where where
P: AsRef<Path>, P: AsRef<Path>,
K: AsRef<str>, K: AsRef<str>,
{ {
if let Some(entry) = index::find(cache.as_ref(), key.as_ref())? { if let Some(entry) = index::find(cache.as_ref(), key.as_ref())? {
data_hash_sync(cache, &entry.integrity) read_hash_sync(cache, &entry.integrity)
} else { } else {
Err(Error::EntryNotFound( Err(Error::EntryNotFound(
cache.as_ref().to_path_buf(), cache.as_ref().to_path_buf(),
@ -377,6 +441,33 @@ where
} }
} }
/// Reads the entire contents of a cache file synchronously into a string,
/// looking the data up by key.
///
/// ## Example
/// ```no_run
/// use anyhow::Result;
/// use std::io::Read;
///
/// fn main() -> Result<()> {
/// let str: String = cacache::read_to_string_sync("./my-cache", "my-key")?;
/// Ok(())
/// }
/// ```
pub fn read_to_string_sync<P, K>(cache: P, key: K) -> Result<String>
where
P: AsRef<Path>,
K: AsRef<str>,
{
if let Some(entry) = index::find(cache.as_ref(), key.as_ref())? {
read_hash_to_string_sync(cache, &entry.integrity)
} else {
Err(Error::EntryNotFound(
cache.as_ref().to_path_buf(),
key.as_ref().into(),
))?
}
}
/// Reads the entire contents of a cache file synchronously into a bytes /// Reads the entire contents of a cache file synchronously into a bytes
/// vector, looking the data up by its content address. /// vector, looking the data up by its content address.
/// ///
@ -386,19 +477,20 @@ where
/// use std::io::Read; /// use std::io::Read;
/// ///
/// fn main() -> Result<()> { /// fn main() -> Result<()> {
/// let sri = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; /// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?;
/// let data = cacache::get::data_hash_sync("./my-cache", &sri)?; /// let data = cacache::read_hash_sync("./my-cache", &sri)?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub fn data_hash_sync<P>(cache: P, sri: &Integrity) -> Result<Vec<u8>> pub fn read_hash_sync<P>(cache: P, sri: &Integrity) -> Result<Vec<u8>>
where where
P: AsRef<Path>, P: AsRef<Path>,
{ {
Ok(read::read(cache.as_ref(), sri)?) Ok(read::read(cache.as_ref(), sri)?)
} }
/// Copies a cache entry by key to a specified location. /// Reads the entire contents of a cache file synchronously into a string,
/// looking the data up by its content address.
/// ///
/// ## Example /// ## Example
/// ```no_run /// ```no_run
@ -406,7 +498,28 @@ where
/// use std::io::Read; /// use std::io::Read;
/// ///
/// fn main() -> Result<()> { /// fn main() -> Result<()> {
/// cacache::get::copy_sync("./my-cache", "my-key", "./my-hello.txt")?; /// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?;
/// let data = cacache::read_hash_sync("./my-cache", &sri)?;
/// Ok(())
/// }
/// ```
pub fn read_hash_to_string_sync<P>(cache: P, sri: &Integrity) -> Result<String>
where
P: AsRef<Path>,
{
Ok(String::from_utf8(read::read(cache.as_ref(), sri)?)?)
}
/// Copies a cache entry by key to a specified location. Returns the number of
/// bytes copied.
///
/// ## Example
/// ```no_run
/// use anyhow::Result;
/// use std::io::Read;
///
/// fn main() -> Result<()> {
/// cacache::copy_sync("./my-cache", "my-key", "./my-hello.txt")?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
@ -426,7 +539,8 @@ where
} }
} }
/// Copies a cache entry by integrity address to a specified location. /// Copies a cache entry by integrity address to a specified location. Returns
/// the number of bytes copied.
/// ///
/// ## Example /// ## Example
/// ```no_run /// ```no_run
@ -434,8 +548,8 @@ where
/// use std::io::Read; /// use std::io::Read;
/// ///
/// fn main() -> Result<()> { /// fn main() -> Result<()> {
/// let sri = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; /// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?;
/// cacache::get::copy_hash_sync("./my-cache", &sri, "./my-hello.txt")?; /// cacache::copy_hash_sync("./my-cache", &sri, "./my-hello.txt")?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
@ -447,8 +561,12 @@ where
read::copy(cache.as_ref(), sri, to.as_ref()) read::copy(cache.as_ref(), sri, to.as_ref())
} }
/// Gets entry information and metadata for a certain key. /// Gets metadata for a certain key.
pub fn entry_sync<P, K>(cache: P, key: K) -> Result<Option<Entry>> ///
/// Note that the existence of a metadata entry is not a guarantee that the
/// underlying data exists, since they are stored and managed independently.
/// To verify that the underlying associated data exists, use `exists_sync()`.
pub fn metadata_sync<P, K>(cache: P, key: K) -> Result<Option<Metadata>>
where where
P: AsRef<Path>, P: AsRef<Path>,
K: AsRef<str>, K: AsRef<str>,
@ -457,49 +575,42 @@ where
} }
/// Returns true if the given hash exists in the cache. /// Returns true if the given hash exists in the cache.
pub fn hash_exists_sync<P: AsRef<Path>>(cache: P, sri: &Integrity) -> bool { pub fn exists_sync<P: AsRef<Path>>(cache: P, sri: &Integrity) -> bool {
read::has_content(cache.as_ref(), &sri).is_some() read::has_content(cache.as_ref(), &sri).is_some()
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use async_attributes;
use async_std::fs as afs;
use async_std::prelude::*; use async_std::prelude::*;
use async_std::{fs as afs, task};
use std::fs; use std::fs;
use tempfile; use tempfile;
#[test] #[async_attributes::test]
fn test_open() { async fn test_open() {
task::block_on(async { let tmp = tempfile::tempdir().unwrap();
let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned();
let dir = tmp.path().to_owned(); crate::write(&dir, "my-key", b"hello world").await.unwrap();
crate::put::data(&dir, "my-key", b"hello world")
.await
.unwrap();
let mut handle = crate::get::open(&dir, "my-key").await.unwrap(); let mut handle = crate::Reader::open(&dir, "my-key").await.unwrap();
let mut str = String::new(); let mut str = String::new();
handle.read_to_string(&mut str).await.unwrap(); handle.read_to_string(&mut str).await.unwrap();
handle.check().unwrap(); handle.check().unwrap();
assert_eq!(str, String::from("hello world")); assert_eq!(str, String::from("hello world"));
});
} }
#[test] #[async_attributes::test]
fn test_open_hash() { async fn test_open_hash() {
task::block_on(async { let tmp = tempfile::tempdir().unwrap();
let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned();
let dir = tmp.path().to_owned(); let sri = crate::write(&dir, "my-key", b"hello world").await.unwrap();
let sri = crate::put::data(&dir, "my-key", b"hello world")
.await
.unwrap();
let mut handle = crate::get::open_hash(&dir, sri).await.unwrap(); let mut handle = crate::Reader::open_hash(&dir, sri).await.unwrap();
let mut str = String::new(); let mut str = String::new();
handle.read_to_string(&mut str).await.unwrap(); handle.read_to_string(&mut str).await.unwrap();
handle.check().unwrap(); handle.check().unwrap();
assert_eq!(str, String::from("hello world")); assert_eq!(str, String::from("hello world"));
});
} }
#[test] #[test]
@ -507,9 +618,9 @@ mod tests {
use std::io::prelude::*; use std::io::prelude::*;
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); crate::write_sync(&dir, "my-key", b"hello world").unwrap();
let mut handle = crate::get::open_sync(&dir, "my-key").unwrap(); let mut handle = crate::SyncReader::open(&dir, "my-key").unwrap();
let mut str = String::new(); let mut str = String::new();
handle.read_to_string(&mut str).unwrap(); handle.read_to_string(&mut str).unwrap();
handle.check().unwrap(); handle.check().unwrap();
@ -521,93 +632,117 @@ mod tests {
use std::io::prelude::*; use std::io::prelude::*;
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri = crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); let sri = crate::write_sync(&dir, "my-key", b"hello world").unwrap();
let mut handle = crate::get::open_hash_sync(&dir, sri).unwrap(); let mut handle = crate::SyncReader::open_hash(&dir, sri).unwrap();
let mut str = String::new(); let mut str = String::new();
handle.read_to_string(&mut str).unwrap(); handle.read_to_string(&mut str).unwrap();
handle.check().unwrap(); handle.check().unwrap();
assert_eq!(str, String::from("hello world")); assert_eq!(str, String::from("hello world"));
} }
#[test] #[async_attributes::test]
fn test_data() { async fn test_read() {
task::block_on(async {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned();
crate::put::data(&dir, "my-key", b"hello world")
.await
.unwrap();
let data = crate::get::data(&dir, "my-key").await.unwrap();
assert_eq!(data, b"hello world");
});
}
#[test]
fn test_data_hash() {
task::block_on(async {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned();
let sri = crate::put::data(&dir, "my-key", b"hello world")
.await
.unwrap();
let data = crate::get::data_hash(&dir, &sri).await.unwrap();
assert_eq!(data, b"hello world");
});
}
#[test]
fn test_data_sync() {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); crate::write(&dir, "my-key", b"hello world").await.unwrap();
let data = crate::get::data_sync(&dir, "my-key").unwrap(); let data = crate::read(&dir, "my-key").await.unwrap();
assert_eq!(data, b"hello world");
}
#[async_attributes::test]
async fn test_read_to_string() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned();
crate::write(&dir, "my-key", "hello world").await.unwrap();
let data = crate::read_to_string(&dir, "my-key").await.unwrap();
assert_eq!(data, "hello world");
}
#[async_attributes::test]
async fn test_read_hash() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned();
let sri = crate::write(&dir, "my-key", b"hello world").await.unwrap();
let data = crate::read_hash(&dir, &sri).await.unwrap();
assert_eq!(data, b"hello world");
}
#[async_attributes::test]
async fn test_read_hash_to_string() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned();
let sri = crate::write(&dir, "my-key", "hello world").await.unwrap();
let data = crate::read_hash_to_string(&dir, &sri).await.unwrap();
assert_eq!(data, "hello world");
}
#[test]
fn test_read_sync() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned();
crate::write_sync(&dir, "my-key", b"hello world").unwrap();
let data = crate::read_sync(&dir, "my-key").unwrap();
assert_eq!(data, b"hello world"); assert_eq!(data, b"hello world");
} }
#[test] #[test]
fn test_data_hash_sync() { fn test_read_to_string_sync() {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri = crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); crate::write_sync(&dir, "my-key", "hello world").unwrap();
let data = crate::get::data_hash_sync(&dir, &sri).unwrap(); let data = crate::read_to_string_sync(&dir, "my-key").unwrap();
assert_eq!(data, "hello world");
}
#[test]
fn test_read_hash_sync() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned();
let sri = crate::write_sync(&dir, "my-key", b"hello world").unwrap();
let data = crate::read_hash_sync(&dir, &sri).unwrap();
assert_eq!(data, b"hello world"); assert_eq!(data, b"hello world");
} }
#[test] #[test]
fn test_copy() { fn test_read_hash_to_string_sync() {
task::block_on(async { let tmp = tempfile::tempdir().unwrap();
let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path().to_owned();
let dir = tmp.path(); let sri = crate::write_sync(&dir, "my-key", "hello world").unwrap();
let dest = dir.join("data");
crate::put::data(&dir, "my-key", b"hello world")
.await
.unwrap();
crate::get::copy(&dir, "my-key", &dest).await.unwrap(); let data = crate::read_hash_to_string_sync(&dir, &sri).unwrap();
let data = afs::read(&dest).await.unwrap(); assert_eq!(data, "hello world");
assert_eq!(data, b"hello world");
});
} }
#[test] #[async_attributes::test]
fn test_copy_hash() { async fn test_copy() {
task::block_on(async { let tmp = tempfile::tempdir().unwrap();
let tmp = tempfile::tempdir().unwrap(); let dir = tmp.path();
let dir = tmp.path(); let dest = dir.join("data");
let dest = dir.join("data"); crate::write(&dir, "my-key", b"hello world").await.unwrap();
let sri = crate::put::data(&dir, "my-key", b"hello world")
.await
.unwrap();
crate::get::copy_hash(&dir, &sri, &dest).await.unwrap(); crate::copy(&dir, "my-key", &dest).await.unwrap();
let data = afs::read(&dest).await.unwrap(); let data = afs::read(&dest).await.unwrap();
assert_eq!(data, b"hello world"); assert_eq!(data, b"hello world");
}); }
#[async_attributes::test]
async fn test_copy_hash() {
let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path();
let dest = dir.join("data");
let sri = crate::write(&dir, "my-key", b"hello world").await.unwrap();
crate::copy_hash(&dir, &sri, &dest).await.unwrap();
let data = afs::read(&dest).await.unwrap();
assert_eq!(data, b"hello world");
} }
#[test] #[test]
@ -615,9 +750,9 @@ mod tests {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path(); let dir = tmp.path();
let dest = dir.join("data"); let dest = dir.join("data");
crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); crate::write_sync(&dir, "my-key", b"hello world").unwrap();
crate::get::copy_sync(&dir, "my-key", &dest).unwrap(); crate::copy_sync(&dir, "my-key", &dest).unwrap();
let data = fs::read(&dest).unwrap(); let data = fs::read(&dest).unwrap();
assert_eq!(data, b"hello world"); assert_eq!(data, b"hello world");
} }
@ -627,9 +762,9 @@ mod tests {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path(); let dir = tmp.path();
let dest = dir.join("data"); let dest = dir.join("data");
let sri = crate::put::data_sync(&dir, "my-key", b"hello world").unwrap(); let sri = crate::write_sync(&dir, "my-key", b"hello world").unwrap();
crate::get::copy_hash_sync(&dir, &sri, &dest).unwrap(); crate::copy_hash_sync(&dir, &sri, &dest).unwrap();
let data = fs::read(&dest).unwrap(); let data = fs::read(&dest).unwrap();
assert_eq!(data, b"hello world"); assert_eq!(data, b"hello world");
} }

View File

@ -21,13 +21,13 @@ use sha2::Sha256;
use ssri::Integrity; use ssri::Integrity;
use walkdir::WalkDir; use walkdir::WalkDir;
use crate::put::PutOpts; use crate::put::WriteOpts;
const INDEX_VERSION: &str = "5"; const INDEX_VERSION: &str = "5";
/// Represents a cache index entry, which points to content. /// Represents a cache index entry, which points to content.
#[derive(PartialEq, Debug)] #[derive(PartialEq, Debug)]
pub struct Entry { pub struct Metadata {
/// Key this entry is stored under. /// Key this entry is stored under.
pub key: String, pub key: String,
/// Integrity hash for the stored data. Acts as a key into {cache}/content. /// Integrity hash for the stored data. Acts as a key into {cache}/content.
@ -36,12 +36,12 @@ pub struct Entry {
pub time: u128, pub time: u128,
/// Size of data associated with this entry. /// Size of data associated with this entry.
pub size: usize, pub size: usize,
/// Arbitrary JSON metadata associated with this entry. /// Arbitrary JSON associated with this entry.
pub metadata: Value, pub metadata: Value,
} }
#[derive(Deserialize, Serialize, Debug)] #[derive(Deserialize, Serialize, Debug)]
struct SerializableEntry { struct SerializableMetadata {
key: String, key: String,
integrity: Option<String>, integrity: Option<String>,
time: u128, time: u128,
@ -49,21 +49,21 @@ struct SerializableEntry {
metadata: Value, metadata: Value,
} }
impl PartialEq for SerializableEntry { impl PartialEq for SerializableMetadata {
fn eq(&self, other: &Self) -> bool { fn eq(&self, other: &Self) -> bool {
self.key == other.key self.key == other.key
} }
} }
impl Eq for SerializableEntry {} impl Eq for SerializableMetadata {}
impl Hash for SerializableEntry { impl Hash for SerializableMetadata {
fn hash<H: Hasher>(&self, state: &mut H) { fn hash<H: Hasher>(&self, state: &mut H) {
self.key.hash(state); self.key.hash(state);
} }
} }
pub fn insert(cache: &Path, key: &str, opts: PutOpts) -> Result<Integrity> { pub fn insert(cache: &Path, key: &str, opts: WriteOpts) -> Result<Integrity> {
let bucket = bucket_path(&cache, &key); let bucket = bucket_path(&cache, &key);
#[cfg(unix)] #[cfg(unix)]
{ {
@ -84,7 +84,7 @@ pub fn insert(cache: &Path, key: &str, opts: PutOpts) -> Result<Integrity> {
bucket.parent().unwrap() bucket.parent().unwrap()
) )
})?; })?;
let stringified = serde_json::to_string(&SerializableEntry { let stringified = serde_json::to_string(&SerializableMetadata {
key: key.to_owned(), key: key.to_owned(),
integrity: opts.sri.clone().map(|x| x.to_string()), integrity: opts.sri.clone().map(|x| x.to_string()),
time: opts.time.unwrap_or_else(now), time: opts.time.unwrap_or_else(now),
@ -112,11 +112,11 @@ pub fn insert(cache: &Path, key: &str, opts: PutOpts) -> Result<Integrity> {
.unwrap()) .unwrap())
} }
pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: PutOpts) -> Result<Integrity> { pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: WriteOpts) -> Result<Integrity> {
let bucket = bucket_path(&cache, &key); let bucket = bucket_path(&cache, &key);
let tmpbucket = bucket.clone(); let tmpbucket = bucket.clone();
#[cfg(unix)] #[cfg(unix)]
let PutOpts { uid, gid, .. } = opts; let WriteOpts { uid, gid, .. } = opts;
task::spawn_blocking(move || { task::spawn_blocking(move || {
let parent = tmpbucket.parent().unwrap(); let parent = tmpbucket.parent().unwrap();
#[cfg(unix)] #[cfg(unix)]
@ -138,7 +138,7 @@ pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: PutOpts) -> R
Ok::<(), anyhow::Error>(()) Ok::<(), anyhow::Error>(())
}) })
.await?; .await?;
let stringified = serde_json::to_string(&SerializableEntry { let stringified = serde_json::to_string(&SerializableMetadata {
key: key.to_owned(), key: key.to_owned(),
integrity: opts.sri.clone().map(|x| x.to_string()), integrity: opts.sri.clone().map(|x| x.to_string()),
time: opts.time.unwrap_or_else(now), time: opts.time.unwrap_or_else(now),
@ -168,7 +168,7 @@ pub async fn insert_async<'a>(cache: &'a Path, key: &'a str, opts: PutOpts) -> R
.unwrap()) .unwrap())
} }
pub fn find(cache: &Path, key: &str) -> Result<Option<Entry>> { pub fn find(cache: &Path, key: &str) -> Result<Option<Metadata>> {
let bucket = bucket_path(cache, &key); let bucket = bucket_path(cache, &key);
Ok(bucket_entries(&bucket) Ok(bucket_entries(&bucket)
.with_context(|| format!("Failed to read index bucket entries from {:?}", bucket))? .with_context(|| format!("Failed to read index bucket entries from {:?}", bucket))?
@ -180,7 +180,7 @@ pub fn find(cache: &Path, key: &str) -> Result<Option<Entry>> {
Ok(sri) => sri, Ok(sri) => sri,
_ => return acc, _ => return acc,
}; };
Some(Entry { Some(Metadata {
key: entry.key, key: entry.key,
integrity, integrity,
size: entry.size, size: entry.size,
@ -196,7 +196,7 @@ pub fn find(cache: &Path, key: &str) -> Result<Option<Entry>> {
})) }))
} }
pub async fn find_async(cache: &Path, key: &str) -> Result<Option<Entry>> { pub async fn find_async(cache: &Path, key: &str) -> Result<Option<Metadata>> {
let bucket = bucket_path(cache, &key); let bucket = bucket_path(cache, &key);
Ok(bucket_entries_async(&bucket) Ok(bucket_entries_async(&bucket)
.await .await
@ -209,7 +209,7 @@ pub async fn find_async(cache: &Path, key: &str) -> Result<Option<Entry>> {
Ok(sri) => sri, Ok(sri) => sri,
_ => return acc, _ => return acc,
}; };
Some(Entry { Some(Metadata {
key: entry.key, key: entry.key,
integrity, integrity,
size: entry.size, size: entry.size,
@ -229,7 +229,7 @@ pub fn delete(cache: &Path, key: &str) -> Result<()> {
insert( insert(
cache, cache,
key, key,
PutOpts { WriteOpts {
algorithm: None, algorithm: None,
size: None, size: None,
sri: None, sri: None,
@ -248,7 +248,7 @@ pub async fn delete_async(cache: &Path, key: &str) -> Result<()> {
insert( insert(
cache, cache,
key, key,
PutOpts { WriteOpts {
algorithm: None, algorithm: None,
size: None, size: None,
sri: None, sri: None,
@ -263,7 +263,7 @@ pub async fn delete_async(cache: &Path, key: &str) -> Result<()> {
.map(|_| ()) .map(|_| ())
} }
pub fn ls(cache: &Path) -> impl Iterator<Item = Result<Entry>> { pub fn ls(cache: &Path) -> impl Iterator<Item = Result<Metadata>> {
WalkDir::new(cache.join(format!("index-v{}", INDEX_VERSION))) WalkDir::new(cache.join(format!("index-v{}", INDEX_VERSION)))
.into_iter() .into_iter()
.map(|bucket| { .map(|bucket| {
@ -274,11 +274,11 @@ pub fn ls(cache: &Path) -> impl Iterator<Item = Result<Entry>> {
Ok(bucket_entries(bucket.path())? Ok(bucket_entries(bucket.path())?
.into_iter() .into_iter()
.collect::<HashSet<SerializableEntry>>() .collect::<HashSet<SerializableMetadata>>()
.into_iter() .into_iter()
.filter_map(|se| { .filter_map(|se| {
if let Some(i) = se.integrity { if let Some(i) = se.integrity {
Some(Entry { Some(Metadata {
key: se.key, key: se.key,
integrity: i.parse().unwrap(), integrity: i.parse().unwrap(),
time: se.time, time: se.time,
@ -325,7 +325,7 @@ fn now() -> u128 {
.as_millis() .as_millis()
} }
fn bucket_entries(bucket: &Path) -> Result<Vec<SerializableEntry>> { fn bucket_entries(bucket: &Path) -> Result<Vec<SerializableMetadata>> {
use std::io::{BufRead, BufReader}; use std::io::{BufRead, BufReader};
fs::File::open(bucket) fs::File::open(bucket)
.map(|file| { .map(|file| {
@ -338,7 +338,7 @@ fn bucket_entries(bucket: &Path) -> Result<Vec<SerializableEntry>> {
// Something's wrong with the entry. Abort. // Something's wrong with the entry. Abort.
_ => return None, _ => return None,
}; };
serde_json::from_str::<SerializableEntry>(entry_str).ok() serde_json::from_str::<SerializableMetadata>(entry_str).ok()
}) })
.collect() .collect()
}) })
@ -351,7 +351,7 @@ fn bucket_entries(bucket: &Path) -> Result<Vec<SerializableEntry>> {
}) })
} }
async fn bucket_entries_async(bucket: &Path) -> Result<Vec<SerializableEntry>> { async fn bucket_entries_async(bucket: &Path) -> Result<Vec<SerializableMetadata>> {
use async_std::io::BufReader; use async_std::io::BufReader;
use futures::io::AsyncBufReadExt; use futures::io::AsyncBufReadExt;
use futures::stream::StreamExt; use futures::stream::StreamExt;
@ -374,7 +374,7 @@ async fn bucket_entries_async(bucket: &Path) -> Result<Vec<SerializableEntry>> {
// Something's wrong with the entry. Abort. // Something's wrong with the entry. Abort.
_ => continue, _ => continue,
}; };
if let Ok(serialized) = serde_json::from_str::<SerializableEntry>(entry_str) { if let Ok(serialized) = serde_json::from_str::<SerializableMetadata>(entry_str) {
vec.push(serialized); vec.push(serialized);
} }
} }
@ -396,7 +396,7 @@ mod tests {
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let sri: Integrity = "sha1-deadbeef".parse().unwrap();
let time = 1_234_567; let time = 1_234_567;
let opts = PutOpts::new().integrity(sri).time(time); let opts = WriteOpts::new().integrity(sri).time(time);
insert(&dir, "hello", opts).unwrap(); insert(&dir, "hello", opts).unwrap();
let entry = std::fs::read_to_string(bucket_path(&dir, "hello")).unwrap(); let entry = std::fs::read_to_string(bucket_path(&dir, "hello")).unwrap();
assert_eq!(entry, MOCK_ENTRY); assert_eq!(entry, MOCK_ENTRY);
@ -408,7 +408,7 @@ mod tests {
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let sri: Integrity = "sha1-deadbeef".parse().unwrap();
let time = 1_234_567; let time = 1_234_567;
let opts = PutOpts::new().integrity(sri).time(time); let opts = WriteOpts::new().integrity(sri).time(time);
task::block_on(async { task::block_on(async {
insert_async(&dir, "hello", opts).await.unwrap(); insert_async(&dir, "hello", opts).await.unwrap();
}); });
@ -428,7 +428,7 @@ mod tests {
let entry = find(&dir, "hello").unwrap().unwrap(); let entry = find(&dir, "hello").unwrap().unwrap();
assert_eq!( assert_eq!(
entry, entry,
Entry { Metadata {
key: String::from("hello"), key: String::from("hello"),
integrity: sri, integrity: sri,
time, time,
@ -451,7 +451,7 @@ mod tests {
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let sri: Integrity = "sha1-deadbeef".parse().unwrap();
let time = 1_234_567; let time = 1_234_567;
let opts = PutOpts::new().integrity(sri).time(time); let opts = WriteOpts::new().integrity(sri).time(time);
insert(&dir, "hello", opts).unwrap(); insert(&dir, "hello", opts).unwrap();
delete(&dir, "hello").unwrap(); delete(&dir, "hello").unwrap();
assert_eq!(find(&dir, "hello").unwrap(), None); assert_eq!(find(&dir, "hello").unwrap(), None);
@ -463,7 +463,7 @@ mod tests {
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let sri: Integrity = "sha1-deadbeef".parse().unwrap();
let time = 1_234_567; let time = 1_234_567;
let opts = PutOpts::new().integrity(sri).time(time); let opts = WriteOpts::new().integrity(sri).time(time);
insert(&dir, "hello", opts).unwrap(); insert(&dir, "hello", opts).unwrap();
task::block_on(async { task::block_on(async {
delete_async(&dir, "hello").await.unwrap(); delete_async(&dir, "hello").await.unwrap();
@ -477,12 +477,12 @@ mod tests {
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let sri: Integrity = "sha1-deadbeef".parse().unwrap();
let time = 1_234_567; let time = 1_234_567;
let opts = PutOpts::new().integrity(sri.clone()).time(time); let opts = WriteOpts::new().integrity(sri.clone()).time(time);
insert(&dir, "hello", opts).unwrap(); insert(&dir, "hello", opts).unwrap();
let entry = find(&dir, "hello").unwrap().unwrap(); let entry = find(&dir, "hello").unwrap().unwrap();
assert_eq!( assert_eq!(
entry, entry,
Entry { Metadata {
key: String::from("hello"), key: String::from("hello"),
integrity: sri, integrity: sri,
time, time,
@ -498,14 +498,14 @@ mod tests {
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let sri: Integrity = "sha1-deadbeef".parse().unwrap();
let time = 1_234_567; let time = 1_234_567;
let opts = PutOpts::new().integrity(sri.clone()).time(time); let opts = WriteOpts::new().integrity(sri.clone()).time(time);
task::block_on(async { task::block_on(async {
insert_async(&dir, "hello", opts).await.unwrap(); insert_async(&dir, "hello", opts).await.unwrap();
}); });
let entry = task::block_on(async { find_async(&dir, "hello").await.unwrap().unwrap() }); let entry = task::block_on(async { find_async(&dir, "hello").await.unwrap().unwrap() });
assert_eq!( assert_eq!(
entry, entry,
Entry { Metadata {
key: String::from("hello"), key: String::from("hello"),
integrity: sri, integrity: sri,
time, time,
@ -521,9 +521,9 @@ mod tests {
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri: Integrity = "sha1-deadbeef".parse().unwrap(); let sri: Integrity = "sha1-deadbeef".parse().unwrap();
let time = 1_234_567; let time = 1_234_567;
let opts = PutOpts::new().integrity(sri.clone()).time(time); let opts = WriteOpts::new().integrity(sri.clone()).time(time);
insert(&dir, "hello", opts).unwrap(); insert(&dir, "hello", opts).unwrap();
let opts = PutOpts::new().integrity(sri).time(time); let opts = WriteOpts::new().integrity(sri).time(time);
insert(&dir, "world", opts).unwrap(); insert(&dir, "world", opts).unwrap();
let mut entries = ls(&dir) let mut entries = ls(&dir)

View File

@ -2,6 +2,31 @@
//! caches. It's really fast, really good at concurrency, and it will never //! caches. It's really fast, really good at concurrency, and it will never
//! give you corrupted data, even if cache files get corrupted or manipulated. //! give you corrupted data, even if cache files get corrupted or manipulated.
//! //!
//! ## API Layout
//!
//! The cacache API is organized roughly similar to `std::fs`; most of the
//! toplevel functionality is available as free functions directly in the
//! `cacache` module, with some additional functionality available through
//! returned objects, as well as `WriteOpts`, which is analogous to
//! `OpenOpts`, but is only able to write.
//!
//! One major difference is that the default APIs are all async functions, as
//! opposed to `std::fs`, where they're all synchronous. Synchronous APIs in
//! cacache are accessible through the `_sync` suffix.
//!
//! ### Suffixes
//!
//! You may notice various suffixes associated with otherwise familiar
//! functions:
//!
//! * `_sync` - Most cacache APIs are asynchronous by default. Anything using
//! the `_sync` suffix behaves just like its unprefixed counterpart, except
//! the operation is synchronous.
//! * `_hash` - Since cacache is a content-addressable cache, the `_hash`
//! suffix means you're interacting directly with content data, skipping the
//! index and its metadata. These functions use an `Integrity` to look up
//! data, instead of a string key.
//!
//! ## Examples //! ## Examples
//! //!
//! Un-suffixed APIs are all async, using //! Un-suffixed APIs are all async, using
@ -15,10 +40,10 @@
//! #[async_attributes::main] //! #[async_attributes::main]
//! async fn main() -> Result<()> { //! async fn main() -> Result<()> {
//! // Data goes in... //! // Data goes in...
//! cacache::put::data("./my-cache", "key", b"hello").await?; //! cacache::write("./my-cache", "key", b"hello").await?;
//! //!
//! // ...data comes out! //! // ...data comes out!
//! let data = cacache::get::data("./my-cache", "key").await?; //! let data = cacache::read("./my-cache", "key").await?;
//! assert_eq!(data, b"hello"); //! assert_eq!(data, b"hello");
//! //!
//! Ok(()) //! Ok(())
@ -40,10 +65,10 @@
//! #[async_attributes::main] //! #[async_attributes::main]
//! async fn main() -> Result<()> { //! async fn main() -> Result<()> {
//! // Data goes in... //! // Data goes in...
//! let sri = cacache::put::data("./my-cache", "key", b"hello").await?; //! let sri = cacache::write("./my-cache", "key", b"hello").await?;
//! //!
//! // ...data gets looked up by `sri` ("Subresource Integrity"). //! // ...data gets looked up by `sri` ("Subresource Integrity").
//! let data = cacache::get::data_hash("./my-cache", &sri).await?; //! let data = cacache::read_hash("./my-cache", &sri).await?;
//! assert_eq!(data, b"hello"); //! assert_eq!(data, b"hello");
//! //!
//! Ok(()) //! Ok(())
@ -62,15 +87,15 @@
//! //!
//! #[async_attributes::main] //! #[async_attributes::main]
//! async fn main() -> Result<()> { //! async fn main() -> Result<()> {
//! let mut fd = cacache::put::PutOpts::new().open("./my-cache", "key").await?; //! let mut fd = cacache::Writer::create("./my-cache", "key").await?;
//! for _ in 0..10 { //! for _ in 0..10 {
//! fd.write_all(b"very large data").await?; //! fd.write_all(b"very large data").await?;
//! } //! }
//! // Data is only persisted to the cache after you do `fd.commit()`! //! // Data is only committed to the cache after you do `fd.commit()`!
//! let sri = fd.commit().await?; //! let sri = fd.commit().await?;
//! println!("integrity: {}", &sri); //! println!("integrity: {}", &sri);
//! //!
//! let mut fd = cacache::get::open("./my-cache", "key").await?; //! let mut fd = cacache::Reader::open("./my-cache", "key").await?;
//! let mut buf = String::new(); //! let mut buf = String::new();
//! fd.read_to_string(&mut buf).await?; //! fd.read_to_string(&mut buf).await?;
//! //!
@ -85,13 +110,17 @@
//! //!
//! ### Sync API //! ### Sync API
//! //!
//! There are also sync APIs available if you don't want to use async/await: //! There are also sync APIs available if you don't want to use async/await.
//! The synchronous APIs are generally faster for linear operations -- that
//! is, doing one thing after another, as opposed to doing many things at
//! once. If you're only reading and writing one thing at a time across your
//! application, you probably want to use these instead.
//! //!
//! ```no_run //! ```no_run
//! use anyhow::Result; //! use anyhow::Result;
//! fn main() -> Result<()> { //! fn main() -> Result<()> {
//! cacache::put::data_sync("./my-cache", "key", b"my-data").unwrap(); //! cacache::write_sync("./my-cache", "key", b"my-data").unwrap();
//! let data = cacache::get::data_sync("./my-cache", "key").unwrap(); //! let data = cacache::read_sync("./my-cache", "key").unwrap();
//! assert_eq!(data, b"my-data"); //! assert_eq!(data, b"my-data");
//! Ok(()) //! Ok(())
//! } //! }
@ -105,10 +134,15 @@ mod content;
mod errors; mod errors;
mod index; mod index;
pub mod get; mod get;
pub mod ls; mod ls;
pub mod put; mod put;
pub mod rm; mod rm;
pub use errors::Error; pub use errors::Error;
pub use index::Entry; pub use index::Metadata;
pub use get::*;
pub use ls::*;
pub use put::*;
pub use rm::*;

View File

@ -4,6 +4,6 @@ use std::path::Path;
use crate::index; use crate::index;
/// Returns a synchronous iterator that lists all cache index entries. /// Returns a synchronous iterator that lists all cache index entries.
pub fn all_sync<P: AsRef<Path>>(cache: P) -> impl Iterator { pub fn list_sync<P: AsRef<Path>>(cache: P) -> impl Iterator {
index::ls(cache.as_ref()) index::ls(cache.as_ref())
} }

View File

@ -26,27 +26,17 @@ use std::task::{Context as TaskContext, Poll};
/// ///
/// #[async_attributes::main] /// #[async_attributes::main]
/// async fn main() -> Result<()> { /// async fn main() -> Result<()> {
/// cacache::put::data("./my-cache", "my-key", b"hello").await?; /// cacache::write("./my-cache", "my-key", b"hello").await?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub async fn data<P, D, K>(cache: P, key: K, data: D) -> Result<Integrity> pub async fn write<P, D, K>(cache: P, key: K, data: D) -> Result<Integrity>
where where
P: AsRef<Path>, P: AsRef<Path>,
D: AsRef<[u8]>, D: AsRef<[u8]>,
K: AsRef<str>, K: AsRef<str>,
{ {
let mut writer = PutOpts::new() let mut writer = Writer::create(cache.as_ref(), key.as_ref()).await?;
.algorithm(Algorithm::Sha256)
.open(cache.as_ref(), key.as_ref())
.await
.with_context(|| {
format!(
"Failed to open a write handle for key {} for cache at {:?}",
key.as_ref(),
cache.as_ref()
)
})?;
writer.write_all(data.as_ref()).await.with_context(|| { writer.write_all(data.as_ref()).await.with_context(|| {
format!( format!(
"Failed to write to cache data for key {} for cache at {:?}", "Failed to write to cache data for key {} for cache at {:?}",
@ -64,15 +54,15 @@ where
} }
/// A reference to an open file writing to the cache. /// A reference to an open file writing to the cache.
pub struct AsyncPut { pub struct Writer {
cache: PathBuf, cache: PathBuf,
key: String, key: String,
written: usize, written: usize,
pub(crate) writer: write::AsyncWriter, pub(crate) writer: write::AsyncWriter,
opts: PutOpts, opts: WriteOpts,
} }
impl AsyncWrite for AsyncPut { impl AsyncWrite for Writer {
fn poll_write( fn poll_write(
mut self: Pin<&mut Self>, mut self: Pin<&mut Self>,
cx: &mut TaskContext<'_>, cx: &mut TaskContext<'_>,
@ -90,8 +80,43 @@ impl AsyncWrite for AsyncPut {
} }
} }
impl AsyncPut { impl Writer {
/// Closes the AsyncPut handle and writes content and index entries. Also /// Creates a new writable file handle into the cache.
///
/// ## Example
/// ```no_run
/// use async_attributes;
/// use async_std::prelude::*;
/// use anyhow::Result;
///
/// #[async_attributes::main]
/// async fn main() -> Result<()> {
/// let mut fd = cacache::Writer::create("./my-cache", "my-key").await?;
/// fd.write_all(b"hello world").await?;
/// // Data is not saved into the cache until you commit it.
/// fd.commit().await?;
/// Ok(())
/// }
/// ```
pub async fn create<P, K>(cache: P, key: K) -> Result<Writer>
where
P: AsRef<Path>,
K: AsRef<str>,
{
WriteOpts::new()
.algorithm(Algorithm::Sha256)
.open(cache.as_ref(), key.as_ref())
.await
.with_context(|| {
format!(
"Failed to open a write handle for key {} for cache at {:?}",
key.as_ref(),
cache.as_ref()
)
})
}
/// Closes the Writer handle and writes content and index entries. Also
/// verifies data against `size` and `integrity` options, if provided. /// verifies data against `size` and `integrity` options, if provided.
/// Must be called manually in order to complete the writing process, /// Must be called manually in order to complete the writing process,
/// otherwise everything will be thrown out. /// otherwise everything will be thrown out.
@ -142,26 +167,17 @@ impl AsyncPut {
/// use std::io::Read; /// use std::io::Read;
/// ///
/// fn main() -> Result<()> { /// fn main() -> Result<()> {
/// let data = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; /// let data = cacache::write_sync("./my-cache", "my-key", b"hello")?;
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub fn data_sync<P, D, K>(cache: P, key: K, data: D) -> Result<Integrity> pub fn write_sync<P, D, K>(cache: P, key: K, data: D) -> Result<Integrity>
where where
P: AsRef<Path>, P: AsRef<Path>,
D: AsRef<[u8]>, D: AsRef<[u8]>,
K: AsRef<str>, K: AsRef<str>,
{ {
let mut writer = PutOpts::new() let mut writer = SyncWriter::create(cache.as_ref(), key.as_ref())?;
.algorithm(Algorithm::Sha256)
.open_sync(cache.as_ref(), key.as_ref())
.with_context(|| {
format!(
"Failed to open a write handle for key {} for cache at {:?}",
key.as_ref(),
cache.as_ref()
)
})?;
writer.write_all(data.as_ref()).with_context(|| { writer.write_all(data.as_ref()).with_context(|| {
format!( format!(
"Failed to write to cache data for key {} for cache at {:?}", "Failed to write to cache data for key {} for cache at {:?}",
@ -180,7 +196,7 @@ where
/// Builder for options and flags for opening a new cache file to write data into. /// Builder for options and flags for opening a new cache file to write data into.
#[derive(Clone, Default)] #[derive(Clone, Default)]
pub struct PutOpts { pub struct WriteOpts {
pub(crate) algorithm: Option<Algorithm>, pub(crate) algorithm: Option<Algorithm>,
pub(crate) sri: Option<Integrity>, pub(crate) sri: Option<Integrity>,
pub(crate) size: Option<usize>, pub(crate) size: Option<usize>,
@ -192,19 +208,19 @@ pub struct PutOpts {
pub(crate) gid: Option<Gid>, pub(crate) gid: Option<Gid>,
} }
impl PutOpts { impl WriteOpts {
/// Creates a blank set of cache writing options. /// Creates a blank set of cache writing options.
pub fn new() -> PutOpts { pub fn new() -> WriteOpts {
Default::default() Default::default()
} }
/// Opens the file handle for writing, returning an AsyncPut instance. /// Opens the file handle for writing, returning an Writer instance.
pub async fn open<P, K>(self, cache: P, key: K) -> Result<AsyncPut> pub async fn open<P, K>(self, cache: P, key: K) -> Result<Writer>
where where
P: AsRef<Path>, P: AsRef<Path>,
K: AsRef<str>, K: AsRef<str>,
{ {
Ok(AsyncPut { Ok(Writer {
cache: cache.as_ref().to_path_buf(), cache: cache.as_ref().to_path_buf(),
key: String::from(key.as_ref()), key: String::from(key.as_ref()),
written: 0, written: 0,
@ -217,13 +233,13 @@ impl PutOpts {
}) })
} }
/// Opens the file handle for writing synchronously, returning a SyncPut instance. /// Opens the file handle for writing synchronously, returning a SyncWriter instance.
pub fn open_sync<P, K>(self, cache: P, key: K) -> Result<SyncPut> pub fn open_sync<P, K>(self, cache: P, key: K) -> Result<SyncWriter>
where where
P: AsRef<Path>, P: AsRef<Path>,
K: AsRef<str>, K: AsRef<str>,
{ {
Ok(SyncPut { Ok(SyncWriter {
cache: cache.as_ref().to_path_buf(), cache: cache.as_ref().to_path_buf(),
key: String::from(key.as_ref()), key: String::from(key.as_ref()),
written: 0, written: 0,
@ -281,15 +297,15 @@ impl PutOpts {
} }
/// A reference to an open file writing to the cache. /// A reference to an open file writing to the cache.
pub struct SyncPut { pub struct SyncWriter {
cache: PathBuf, cache: PathBuf,
key: String, key: String,
written: usize, written: usize,
pub(crate) writer: write::Writer, pub(crate) writer: write::Writer,
opts: PutOpts, opts: WriteOpts,
} }
impl Write for SyncPut { impl Write for SyncWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.writer.write(buf) self.writer.write(buf)
} }
@ -298,8 +314,40 @@ impl Write for SyncPut {
} }
} }
impl SyncPut { impl SyncWriter {
/// Closes the Put handle and writes content and index entries. Also /// Creates a new writable file handle into the cache.
///
/// ## Example
/// ```no_run
/// use anyhow::Result;
/// use std::io::prelude::*;
///
/// fn main() -> Result<()> {
/// let mut fd = cacache::SyncWriter::create("./my-cache", "my-key")?;
/// fd.write_all(b"hello world")?;
/// // Data is not saved into the cache until you commit it.
/// fd.commit()?;
/// Ok(())
/// }
/// ```
pub fn create<P, K>(cache: P, key: K) -> Result<SyncWriter>
where
P: AsRef<Path>,
K: AsRef<str>,
{
WriteOpts::new()
.algorithm(Algorithm::Sha256)
.open_sync(cache.as_ref(), key.as_ref())
.with_context(|| {
format!(
"Failed to open a write handle for key {} for cache at {:?}",
key.as_ref(),
cache.as_ref()
)
})
}
/// Closes the Writer handle and writes content and index entries. Also
/// verifies data against `size` and `integrity` options, if provided. /// verifies data against `size` and `integrity` options, if provided.
/// Must be called manually in order to complete the writing process, /// Must be called manually in order to complete the writing process,
/// otherwise everything will be thrown out. /// otherwise everything will be thrown out.
@ -343,16 +391,14 @@ impl SyncPut {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use async_std::task; use async_attributes;
#[test] #[async_attributes::test]
fn round_trip() { async fn round_trip() {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
task::block_on(async { crate::write(&dir, "hello", b"hello").await.unwrap();
crate::put::data(&dir, "hello", b"hello").await.unwrap(); let data = crate::read(&dir, "hello").await.unwrap();
});
let data = task::block_on(async { crate::get::data(&dir, "hello").await.unwrap() });
assert_eq!(data, b"hello"); assert_eq!(data, b"hello");
} }
@ -360,8 +406,8 @@ mod tests {
fn round_trip_sync() { fn round_trip_sync() {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
crate::put::data_sync(&dir, "hello", b"hello").unwrap(); crate::write_sync(&dir, "hello", b"hello").unwrap();
let data = crate::get::data_sync(&dir, "hello").unwrap(); let data = crate::read_sync(&dir, "hello").unwrap();
assert_eq!(data, b"hello"); assert_eq!(data, b"hello");
} }
} }

139
src/rm.rs
View File

@ -10,8 +10,8 @@ use ssri::Integrity;
use crate::content::rm; use crate::content::rm;
use crate::index; use crate::index;
/// Removes an individual index entry. The associated content will be left /// Removes an individual index metadata entry. The associated content will be
/// intact. /// left in the cache.
/// ///
/// ## Example /// ## Example
/// ```no_run /// ```no_run
@ -21,19 +21,20 @@ use crate::index;
/// ///
/// #[async_attributes::main] /// #[async_attributes::main]
/// async fn main() -> Result<()> { /// async fn main() -> Result<()> {
/// let sri = cacache::put::data("./my-cache", "my-key", b"hello").await?; /// let sri = cacache::write("./my-cache", "my-key", b"hello").await?;
/// ///
/// cacache::rm::entry("./my-cache", "my-key").await?; /// cacache::remove("./my-cache", "my-key").await?;
/// ///
/// // This fails: /// // This fails:
/// cacache::get::data("./my-cache", "my-key").await?; /// cacache::read("./my-cache", "my-key").await?;
/// ///
/// // But this succeeds: /// // But this succeeds:
/// cacache::get::data_hash("./my-cache", &sri).await?; /// cacache::read_hash("./my-cache", &sri).await?;
///
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub async fn entry<P, K>(cache: P, key: K) -> Result<()> pub async fn remove<P, K>(cache: P, key: K) -> Result<()>
where where
P: AsRef<Path>, P: AsRef<Path>,
K: AsRef<str>, K: AsRef<str>,
@ -60,21 +61,21 @@ where
/// ///
/// #[async_attributes::main] /// #[async_attributes::main]
/// async fn main() -> Result<()> { /// async fn main() -> Result<()> {
/// let sri = cacache::put::data("./my-cache", "my-key", b"hello").await?; /// let sri = cacache::write("./my-cache", "my-key", b"hello").await?;
/// ///
/// cacache::rm::entry("./my-cache", "my-key").await?; /// cacache::remove_hash("./my-cache", &sri).await?;
/// ///
/// // These fail: /// // These fail:
/// cacache::get::data("./my-cache", "my-key").await?; /// cacache::read("./my-cache", "my-key").await?;
/// cacache::get::data_hash("./my-cache", &sri).await?; /// cacache::read_hash("./my-cache", &sri).await?;
/// ///
/// // But this succeeds: /// // But this succeeds:
/// cacache::get::entry("./my-cache", "my-key").await?; /// cacache::metadata("./my-cache", "my-key").await?;
/// ///
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub async fn content<P: AsRef<Path>>(cache: P, sri: &Integrity) -> Result<()> { pub async fn remove_hash<P: AsRef<Path>>(cache: P, sri: &Integrity) -> Result<()> {
rm::rm_async(cache.as_ref(), &sri).await.with_context(|| { rm::rm_async(cache.as_ref(), &sri).await.with_context(|| {
format!( format!(
"Failed to remove content under {} in cache at {:?}", "Failed to remove content under {} in cache at {:?}",
@ -95,19 +96,19 @@ pub async fn content<P: AsRef<Path>>(cache: P, sri: &Integrity) -> Result<()> {
/// ///
/// #[async_attributes::main] /// #[async_attributes::main]
/// async fn main() -> Result<()> { /// async fn main() -> Result<()> {
/// let sri = cacache::put::data("./my-cache", "my-key", b"hello").await?; /// let sri = cacache::write("./my-cache", "my-key", b"hello").await?;
/// ///
/// cacache::rm::entry("./my-cache", "my-key").await?; /// cacache::clear("./my-cache").await?;
/// ///
/// // These all fail: /// // These all fail:
/// cacache::get::data("./my-cache", "my-key").await?; /// cacache::read("./my-cache", "my-key").await?;
/// cacache::get::entry("./my-cache", "my-key").await?; /// cacache::metadata("./my-cache", "my-key").await?;
/// cacache::get::data_hash("./my-cache", &sri).await?; /// cacache::read_hash("./my-cache", &sri).await?;
/// ///
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub async fn all<P: AsRef<Path>>(cache: P) -> Result<()> { pub async fn clear<P: AsRef<Path>>(cache: P) -> Result<()> {
for entry in cache.as_ref().read_dir()? { for entry in cache.as_ref().read_dir()? {
if let Ok(entry) = entry { if let Ok(entry) = entry {
afs::remove_dir_all(entry.path()).await?; afs::remove_dir_all(entry.path()).await?;
@ -117,7 +118,7 @@ pub async fn all<P: AsRef<Path>>(cache: P) -> Result<()> {
} }
/// Removes an individual index entry synchronously. The associated content /// Removes an individual index entry synchronously. The associated content
/// will be left intact. /// will be left in the cache.
/// ///
/// ## Example /// ## Example
/// ```no_run /// ```no_run
@ -125,20 +126,20 @@ pub async fn all<P: AsRef<Path>>(cache: P) -> Result<()> {
/// use std::io::Read; /// use std::io::Read;
/// ///
/// fn main() -> Result<()> { /// fn main() -> Result<()> {
/// let sri = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; /// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?;
/// ///
/// cacache::rm::entry_sync("./my-cache", "my-key")?; /// cacache::remove_sync("./my-cache", "my-key")?;
/// ///
/// // This fails: /// // This fails:
/// cacache::get::data_sync("./my-cache", "my-key")?; /// cacache::read_sync("./my-cache", "my-key")?;
/// ///
/// // But this succeeds: /// // But this succeeds:
/// cacache::get::data_hash_sync("./my-cache", &sri)?; /// cacache::read_hash_sync("./my-cache", &sri)?;
/// ///
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub fn entry_sync<P, K>(cache: P, key: K) -> Result<()> pub fn remove_sync<P, K>(cache: P, key: K) -> Result<()>
where where
P: AsRef<Path>, P: AsRef<Path>,
K: AsRef<str>, K: AsRef<str>,
@ -161,21 +162,21 @@ where
/// use std::io::Read; /// use std::io::Read;
/// ///
/// fn main() -> Result<()> { /// fn main() -> Result<()> {
/// let sri = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; /// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?;
/// ///
/// cacache::rm::entry_sync("./my-cache", "my-key")?; /// cacache::remove_hash_sync("./my-cache", &sri)?;
/// ///
/// // These fail: /// // These fail:
/// cacache::get::data_sync("./my-cache", "my-key")?; /// cacache::read_sync("./my-cache", "my-key")?;
/// cacache::get::data_hash_sync("./my-cache", &sri)?; /// cacache::read_hash_sync("./my-cache", &sri)?;
/// ///
/// // But this succeeds: /// // But this succeeds:
/// cacache::get::entry_sync("./my-cache", "my-key")?; /// cacache::metadata_sync("./my-cache", "my-key")?;
/// ///
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub fn content_sync<P: AsRef<Path>>(cache: P, sri: &Integrity) -> Result<()> { pub fn remove_hash_sync<P: AsRef<Path>>(cache: P, sri: &Integrity) -> Result<()> {
rm::rm(cache.as_ref(), &sri).with_context(|| { rm::rm(cache.as_ref(), &sri).with_context(|| {
format!( format!(
"Failed to remove content under {} in cache at {:?}", "Failed to remove content under {} in cache at {:?}",
@ -194,19 +195,19 @@ pub fn content_sync<P: AsRef<Path>>(cache: P, sri: &Integrity) -> Result<()> {
/// use std::io::Read; /// use std::io::Read;
/// ///
/// fn main() -> Result<()> { /// fn main() -> Result<()> {
/// let sri = cacache::put::data_sync("./my-cache", "my-key", b"hello")?; /// let sri = cacache::write_sync("./my-cache", "my-key", b"hello")?;
/// ///
/// cacache::rm::entry_sync("./my-cache", "my-key")?; /// cacache::clear_sync("./my-cache")?;
/// ///
/// // These all fail: /// // These all fail:
/// cacache::get::data_sync("./my-cache", "my-key")?; /// cacache::read_sync("./my-cache", "my-key")?;
/// cacache::get::data_hash_sync("./my-cache", &sri)?; /// cacache::read_hash_sync("./my-cache", &sri)?;
/// cacache::get::entry_sync("./my-cache", "my-key")?; /// cacache::metadata_sync("./my-cache", "my-key")?;
/// ///
/// Ok(()) /// Ok(())
/// } /// }
/// ``` /// ```
pub fn all_sync<P: AsRef<Path>>(cache: P) -> Result<()> { pub fn clear_sync<P: AsRef<Path>>(cache: P) -> Result<()> {
for entry in cache.as_ref().read_dir()? { for entry in cache.as_ref().read_dir()? {
if let Ok(entry) = entry { if let Ok(entry) = entry {
fs::remove_dir_all(entry.path())?; fs::remove_dir_all(entry.path())?;
@ -220,98 +221,98 @@ mod tests {
use async_std::task; use async_std::task;
#[test] #[test]
fn entry() { fn test_remove() {
task::block_on(async { task::block_on(async {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri = crate::put::data(&dir, "key", b"my-data").await.unwrap(); let sri = crate::write(&dir, "key", b"my-data").await.unwrap();
crate::rm::entry(&dir, "key").await.unwrap(); crate::remove(&dir, "key").await.unwrap();
let entry = crate::get::entry(&dir, "key").await.unwrap(); let entry = crate::metadata(&dir, "key").await.unwrap();
assert_eq!(entry, None); assert_eq!(entry, None);
let data_exists = crate::get::hash_exists(&dir, &sri).await; let data_exists = crate::exists(&dir, &sri).await;
assert_eq!(data_exists, true); assert_eq!(data_exists, true);
}); });
} }
#[test] #[test]
fn content() { fn test_remove_data() {
task::block_on(async { task::block_on(async {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri = crate::put::data(&dir, "key", b"my-data").await.unwrap(); let sri = crate::write(&dir, "key", b"my-data").await.unwrap();
crate::rm::content(&dir, &sri).await.unwrap(); crate::remove_hash(&dir, &sri).await.unwrap();
let entry = crate::get::entry(&dir, "key").await.unwrap(); let entry = crate::metadata(&dir, "key").await.unwrap();
assert_eq!(entry.is_some(), true); assert_eq!(entry.is_some(), true);
let data_exists = crate::get::hash_exists(&dir, &sri).await; let data_exists = crate::exists(&dir, &sri).await;
assert_eq!(data_exists, false); assert_eq!(data_exists, false);
}); });
} }
#[test] #[test]
fn all() { fn test_clear() {
task::block_on(async { task::block_on(async {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri = crate::put::data(&dir, "key", b"my-data").await.unwrap(); let sri = crate::write(&dir, "key", b"my-data").await.unwrap();
crate::rm::all(&dir).await.unwrap(); crate::clear(&dir).await.unwrap();
let entry = crate::get::entry(&dir, "key").await.unwrap(); let entry = crate::metadata(&dir, "key").await.unwrap();
assert_eq!(entry.is_some(), false); assert_eq!(entry.is_some(), false);
let data_exists = crate::get::hash_exists(&dir, &sri).await; let data_exists = crate::exists(&dir, &sri).await;
assert_eq!(data_exists, false); assert_eq!(data_exists, false);
}); });
} }
#[test] #[test]
fn entry_sync() { fn test_remove_sync() {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri = crate::put::data_sync(&dir, "key", b"my-data").unwrap(); let sri = crate::write_sync(&dir, "key", b"my-data").unwrap();
crate::rm::entry_sync(&dir, "key").unwrap(); crate::remove_sync(&dir, "key").unwrap();
let new_entry = crate::get::entry_sync(&dir, "key").unwrap(); let new_entry = crate::metadata_sync(&dir, "key").unwrap();
assert_eq!(new_entry, None); assert_eq!(new_entry, None);
let data_exists = crate::get::hash_exists_sync(&dir, &sri); let data_exists = crate::exists_sync(&dir, &sri);
assert_eq!(data_exists, true); assert_eq!(data_exists, true);
} }
#[test] #[test]
fn content_sync() { fn test_remove_data_sync() {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri = crate::put::data_sync(&dir, "key", b"my-data").unwrap(); let sri = crate::write_sync(&dir, "key", b"my-data").unwrap();
crate::rm::content_sync(&dir, &sri).unwrap(); crate::remove_hash_sync(&dir, &sri).unwrap();
let new_entry = crate::get::entry_sync(&dir, "key").unwrap(); let entry = crate::metadata_sync(&dir, "key").unwrap();
assert_eq!(new_entry.is_some(), true); assert_eq!(entry.is_some(), true);
let data_exists = crate::get::hash_exists_sync(&dir, &sri); let data_exists = crate::exists_sync(&dir, &sri);
assert_eq!(data_exists, false); assert_eq!(data_exists, false);
} }
#[test] #[test]
fn all_sync() { fn test_clear_sync() {
let tmp = tempfile::tempdir().unwrap(); let tmp = tempfile::tempdir().unwrap();
let dir = tmp.path().to_owned(); let dir = tmp.path().to_owned();
let sri = crate::put::data_sync(&dir, "key", b"my-data").unwrap(); let sri = crate::write_sync(&dir, "key", b"my-data").unwrap();
crate::rm::all_sync(&dir).unwrap(); crate::clear_sync(&dir).unwrap();
let new_entry = crate::get::entry_sync(&dir, "key").unwrap(); let entry = crate::metadata_sync(&dir, "key").unwrap();
assert_eq!(new_entry, None); assert_eq!(entry, None);
let data_exists = crate::get::hash_exists_sync(&dir, &sri); let data_exists = crate::exists_sync(&dir, &sri);
assert_eq!(data_exists, false); assert_eq!(data_exists, false);
} }
} }