From b0f351ea269778e2e0be1d1388698d7a4b97ccd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kat=20March=C3=A1n?= Date: Wed, 5 Jun 2019 17:07:48 +0200 Subject: [PATCH] feat(ls): implemented cacache::ls::all() --- Cargo.lock | 37 +++++++++++++++++++++++++++++ Cargo.toml | 2 ++ src/errors.rs | 9 +++++++ src/index.rs | 66 +++++++++++++++++++++++++++++++++++++++++++++++---- src/lib.rs | 1 + src/ls.rs | 8 +++++++ 6 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 src/ls.rs diff --git a/Cargo.lock b/Cargo.lock index 8265e77..e3dba31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,6 +85,7 @@ dependencies = [ "atomicwrites 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "chownr 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "mkdirp 1.0.0 (git+https://github.com/yoshuawuyts/mkdirp)", @@ -96,6 +97,7 @@ dependencies = [ "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "ssri 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -133,6 +135,11 @@ dependencies = [ "generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "either" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "failure" version = "0.1.5" @@ -375,6 +382,14 @@ name = "ryu" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "same-file" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "serde" version = "1.0.92" @@ -494,6 +509,16 @@ name = "void" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "walkdir" +version = "2.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "same-file 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "winapi" version = "0.3.7" @@ -508,6 +533,14 @@ name = "winapi-i686-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "winapi-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -529,6 +562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum chownr 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "53acee514702cea2b35b2a2ab596fed5a1fbf03bd3800b9a515cf727e41e1dc9" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05f47366984d3ad862010e22c7ce81a7dbcaebbdfb37241a620f8b6596ee135c" +"checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" @@ -559,6 +593,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum rustc-demangle 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "ccc78bfd5acd7bf3e89cffcf899e5cb1a52d6fafa8dec2739ad70c9577a57288" "checksum ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "b96a9549dc8d48f2c283938303c4b5a77aa29bfbc5b54b084fb1630408899a8f" +"checksum same-file 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8f20c4be53a8a1ff4c1f1b2bd14570d2f634628709752f0702ecdd2b3f9a5267" "checksum serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "32746bf0f26eab52f06af0d0aa1984f641341d06d8d673c693871da2d188c9be" "checksum serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "46a3223d0c9ba936b61c0d2e3e559e3217dbfb8d65d06d26e8b3c25de38bae3e" "checksum serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" @@ -572,6 +607,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +"checksum walkdir 2.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9d9d7ed3431229a144296213105a390676cc49c9b6a72bd19f3176c98e129fa1" "checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/Cargo.toml b/Cargo.toml index c500160..9d49efb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,5 @@ nix = "0.14.0" mkdirp = { git = "https://github.com/yoshuawuyts/mkdirp" } chownr = "2.0.0" failure = "0.1.5" +walkdir = "2.2.7" +either = "1.5.2" diff --git a/src/errors.rs b/src/errors.rs index d167b8c..ca8b25d 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -4,6 +4,7 @@ use atomicwrites; use chownr; use failure::Fail; use serde_json; +use walkdir; /// Error type returned by all API calls. #[derive(Fail, Debug)] @@ -22,6 +23,8 @@ pub enum Error { SerdeJson(#[fail(cause)] serde_json::error::Error), #[fail(display = "{}", _0)] AtomicWrite(#[fail(cause)] atomicwrites::Error), + #[fail(display = "{}", _0)] + WalkDir(#[fail(cause)] walkdir::Error), } impl From for Error { @@ -47,3 +50,9 @@ impl From> for Error { Error::AtomicWrite(error) } } + +impl From for Error { + fn from(error: walkdir::Error) -> Self { + Error::WalkDir(error) + } +} diff --git a/src/index.rs b/src/index.rs index 186db2b..3c162fd 100644 --- a/src/index.rs +++ b/src/index.rs @@ -1,3 +1,4 @@ +use std::collections::hash_map::HashMap; use std::fs::{self, OpenOptions}; use std::io::{ErrorKind, Write}; use std::path::{Path, PathBuf}; @@ -5,6 +6,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use chownr; use digest::Digest; +use either::{Left, Right}; use hex; use mkdirp; use serde_derive::{Deserialize, Serialize}; @@ -12,6 +14,7 @@ use serde_json::{json, Value}; use sha1::Sha1; use sha2::Sha256; use ssri::Integrity; +use walkdir::WalkDir; use crate::put::Writer; use crate::errors::Error; @@ -99,10 +102,42 @@ pub fn delete(cache: &Path, key: &str) -> Result<(), Error> { Ok(()) } -// TODO -// pub fn ls(_cache: &Path) { -// unimplemented!(); -// } +pub fn ls(cache: &Path) -> impl Iterator> { + let mut path = PathBuf::new(); + path.push(cache); + path.push(format!("index-v{}", INDEX_VERSION)); + WalkDir::new(path).into_iter().map(|bucket| { + let bucket = bucket?; + if bucket.file_type().is_dir() { + return Ok(core::iter::empty().collect::>()) + } + let entries = bucket_entries(bucket.path())?; + let mut dedupe: HashMap = HashMap::new(); + for entry in entries { + dedupe.insert(entry.key.clone(), entry); + } + let iter = dedupe + .into_iter() + .filter(|se| se.1.integrity.is_some()) + .map(|se| { + let se = se.1; + Entry { + key: se.key, + integrity: se.integrity.unwrap().parse().unwrap(), + time: se.time, + size: se.size, + metadata: se.metadata, + } + }); + Ok(iter.collect::>()) + }) + .flat_map(|res| { + match res { + Ok(it) => Left(it.into_iter().map(Ok)), + Err(err) => Right(std::iter::once(Err(err))) + } + }) +} fn bucket_path(cache: &Path, key: &str) -> PathBuf { let hashed = hash_key(&key); @@ -225,4 +260,27 @@ mod tests { delete(&dir, "hello").unwrap(); assert_eq!(find(&dir, "hello").unwrap(), None); } + + #[test] + fn ls_basic() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().to_owned(); + let sri: Integrity = "sha1-deadbeef".parse().unwrap(); + let time = 1_234_567; + let writer = Writer::new(&dir, "hello") + .integrity(sri.clone()) + .time(time); + insert(writer).unwrap(); + let writer = Writer::new(&dir, "world") + .integrity(sri.clone()) + .time(time); + insert(writer).unwrap(); + + let mut entries = ls(&dir) + .map(|x| Ok(x?.key)) + .collect::, Error>>() + .unwrap(); + entries.sort(); + assert_eq!(entries, vec![String::from("hello"), String::from("world")]) + } } diff --git a/src/lib.rs b/src/lib.rs index a0d5a2e..05b0ec5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ mod index; pub mod get; pub mod put; pub mod rm; +pub mod ls; pub use errors::Error; pub use index::Entry; diff --git a/src/ls.rs b/src/ls.rs new file mode 100644 index 0000000..73e5938 --- /dev/null +++ b/src/ls.rs @@ -0,0 +1,8 @@ +//! Functions for iterating over the cache. +use std::path::Path; + +use crate::index; + +pub fn all(cache: &Path) -> impl Iterator { + index::ls(cache) +}