Support serving pre-compressed files for static sites (#2615)

* support serving pre-compressed files for static sites

* Update CHANGES.md

* fix behavior change for audio file

* follow-up some inconsistency

* test(files): make encoding test independent of fixture line endings

---------

Co-authored-by: Rob Ede <robjtede@icloud.com>
Co-authored-by: Yuki Okushi <huyuumi.dev@gmail.com>

actix-files/CHANGES.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+- Add `Files::try_compressed()` to support serving pre-compressed static files [#2615]
+
+[#2615]: https://github.com/actix/actix-web/pull/2615
+
 ## 0.6.10
 
 ### Security Notice

actix-files/src/files.rs

@@ -50,6 +50,7 @@ pub struct Files {
     use_guards: Option<Rc<dyn Guard>>,
     guards: Vec<Rc<dyn Guard>>,
     hidden_files: bool,
+    try_compressed: bool,
     read_mode_threshold: u64,
 }
 
@@ -76,6 +77,7 @@ impl Clone for Files {
             use_guards: self.use_guards.clone(),
             guards: self.guards.clone(),
             hidden_files: self.hidden_files,
+            try_compressed: self.try_compressed,
             read_mode_threshold: self.read_mode_threshold,
         }
     }
@@ -128,6 +130,7 @@ impl Files {
             use_guards: None,
             guards: Vec::new(),
             hidden_files: false,
+            try_compressed: false,
             read_mode_threshold: 0,
         }
     }
@@ -351,6 +354,15 @@ impl Files {
         self.hidden_files = true;
         self
     }
+
+    /// Attempts to search for a suitable pre-compressed version of a file on disk before falling
+    /// back to the uncompressed version.
+    ///
+    /// Currently, `.gz`, `.br`, and `.zst` files are supported.
+    pub fn try_compressed(mut self) -> Self {
+        self.try_compressed = true;
+        self
+    }
 }
 
 impl HttpServiceFactory for Files {
@@ -402,6 +414,7 @@ impl ServiceFactory<ServiceRequest> for Files {
             file_flags: self.file_flags,
             guards: self.use_guards.clone(),
             hidden_files: self.hidden_files,
+            try_compressed: self.try_compressed,
             size_threshold: self.read_mode_threshold,
             with_permanent_redirect: self.with_permanent_redirect,
         };

actix-files/src/named.rs

@@ -91,6 +91,55 @@ pub(crate) use tokio_uring::fs::File;
 
 use super::chunked;
 
+pub(crate) fn get_content_type_and_disposition(
+    path: &Path,
+) -> Result<(mime::Mime, ContentDisposition), io::Error> {
+    let filename = match path.file_name() {
+        Some(name) => name.to_string_lossy(),
+        None => {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "Provided path has no filename",
+            ));
+        }
+    };
+
+    let ct = mime_guess::from_path(path).first_or_octet_stream();
+
+    let disposition = match ct.type_() {
+        mime::IMAGE | mime::TEXT | mime::AUDIO | mime::VIDEO => DispositionType::Inline,
+        mime::APPLICATION => match ct.subtype() {
+            mime::JAVASCRIPT | mime::JSON => DispositionType::Inline,
+            name if name == "wasm" || name == "xhtml" => DispositionType::Inline,
+            _ => DispositionType::Attachment,
+        },
+        _ => DispositionType::Attachment,
+    };
+
+    // replace special characters in filenames which could occur on some filesystems
+    let filename_s = filename
+        .replace('\n', "%0A") // \n line break
+        .replace('\x0B', "%0B") // \v vertical tab
+        .replace('\x0C', "%0C") // \f form feed
+        .replace('\r', "%0D"); // \r carriage return
+    let mut parameters = vec![DispositionParam::Filename(filename_s)];
+
+    if !filename.is_ascii() {
+        parameters.push(DispositionParam::FilenameExt(ExtendedValue {
+            charset: Charset::Ext(String::from("UTF-8")),
+            language_tag: None,
+            value: filename.into_owned().into_bytes(),
+        }))
+    }
+
+    let cd = ContentDisposition {
+        disposition,
+        parameters,
+    };
+
+    Ok((ct, cd))
+}
+
 impl NamedFile {
     /// Creates an instance from a previously opened file.
     ///
@@ -117,52 +166,7 @@ impl NamedFile {
 
         // Get the name of the file and use it to construct default Content-Type
         // and Content-Disposition values
-        let (content_type, content_disposition) = {
-            let filename = match path.file_name() {
-                Some(name) => name.to_string_lossy(),
-                None => {
-                    return Err(io::Error::new(
-                        io::ErrorKind::InvalidInput,
-                        "Provided path has no filename",
-                    ));
-                }
-            };
-
-            let ct = mime_guess::from_path(&path).first_or_octet_stream();
-
-            let disposition = match ct.type_() {
-                mime::IMAGE | mime::TEXT | mime::AUDIO | mime::VIDEO => DispositionType::Inline,
-                mime::APPLICATION => match ct.subtype() {
-                    mime::JAVASCRIPT | mime::JSON => DispositionType::Inline,
-                    name if name == "wasm" || name == "xhtml" => DispositionType::Inline,
-                    _ => DispositionType::Attachment,
-                },
-                _ => DispositionType::Attachment,
-            };
-
-            // replace special characters in filenames which could occur on some filesystems
-            let filename_s = filename
-                .replace('\n', "%0A") // \n line break
-                .replace('\x0B', "%0B") // \v vertical tab
-                .replace('\x0C', "%0C") // \f form feed
-                .replace('\r', "%0D"); // \r carriage return
-            let mut parameters = vec![DispositionParam::Filename(filename_s)];
-
-            if !filename.is_ascii() {
-                parameters.push(DispositionParam::FilenameExt(ExtendedValue {
-                    charset: Charset::Ext(String::from("UTF-8")),
-                    language_tag: None,
-                    value: filename.into_owned().into_bytes(),
-                }))
-            }
-
-            let cd = ContentDisposition {
-                disposition,
-                parameters,
-            };
-
-            (ct, cd)
-        };
+        let (content_type, content_disposition) = get_content_type_and_disposition(&path)?;
 
         let md = {
             #[cfg(not(feature = "experimental-io-uring"))]
@@ -710,3 +714,14 @@ impl HttpServiceFactory for NamedFile {
         )
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn audio_files_use_inline_content_disposition() {
+        let (_ct, cd) = get_content_type_and_disposition(Path::new("sound.mp3")).unwrap();
+        assert_eq!(cd.disposition, DispositionType::Inline);
+    }
+}

actix-files/src/service.rs

@@ -1,4 +1,9 @@
-use std::{fmt, io, ops::Deref, path::PathBuf, rc::Rc};
+use std::{
+    fmt, io,
+    ops::Deref,
+    path::{Path, PathBuf},
+    rc::Rc,
+};
 
 use actix_web::{
     body::BoxBody,
@@ -39,6 +44,7 @@ pub struct FilesServiceInner {
     pub(crate) file_flags: named::Flags,
     pub(crate) guards: Option<Rc<dyn Guard>>,
     pub(crate) hidden_files: bool,
+    pub(crate) try_compressed: bool,
     pub(crate) size_threshold: u64,
     pub(crate) with_permanent_redirect: bool,
 }
@@ -64,7 +70,12 @@ impl FilesService {
         }
     }
 
-    fn serve_named_file(&self, req: ServiceRequest, mut named_file: NamedFile) -> ServiceResponse {
+    fn serve_named_file_with_encoding(
+        &self,
+        req: ServiceRequest,
+        mut named_file: NamedFile,
+        encoding: header::ContentEncoding,
+    ) -> ServiceResponse {
         if let Some(ref mime_override) = self.mime_override {
             let new_disposition = mime_override(&named_file.content_type.type_());
             named_file.content_disposition.disposition = new_disposition;
@@ -72,12 +83,36 @@ impl FilesService {
         named_file.flags = self.file_flags;
 
         let (req, _) = req.into_parts();
-        let res = named_file
+        let mut res = named_file
             .read_mode_threshold(self.size_threshold)
             .into_response(&req);
+
+        let header_value = match encoding {
+            header::ContentEncoding::Brotli => Some("br"),
+            header::ContentEncoding::Gzip => Some("gzip"),
+            header::ContentEncoding::Zstd => Some("zstd"),
+            header::ContentEncoding::Identity => None,
+            // Only variants in SUPPORTED_PRECOMPRESSION_ENCODINGS can occur here
+            _ => unreachable!(),
+        };
+        if let Some(header_value) = header_value {
+            res.headers_mut().insert(
+                header::CONTENT_ENCODING,
+                header::HeaderValue::from_static(header_value),
+            );
+            // Response representation varies by Accept-Encoding when serving pre-compressed assets.
+            res.headers_mut().append(
+                header::VARY,
+                header::HeaderValue::from_static("accept-encoding"),
+            );
+        }
         ServiceResponse::new(req, res)
     }
 
+    fn serve_named_file(&self, req: ServiceRequest, named_file: NamedFile) -> ServiceResponse {
+        self.serve_named_file_with_encoding(req, named_file, header::ContentEncoding::Identity)
+    }
+
     fn show_index(&self, req: ServiceRequest, path: PathBuf) -> ServiceResponse {
         let dir = Directory::new(self.directory.clone(), path);
 
@@ -138,6 +173,15 @@ impl Service<ServiceRequest> for FilesService {
 
             // full file path
             let path = this.directory.join(&path_on_disk);
+
+            // Try serving pre-compressed file even if the uncompressed file doesn't exist yet.
+            // Still handle directories (index/listing) through the normal branch below.
+            if this.try_compressed && !path.is_dir() {
+                if let Some((named_file, encoding)) = find_compressed(&req, &path).await {
+                    return Ok(this.serve_named_file_with_encoding(req, named_file, encoding));
+                }
+            }
+
             if let Err(err) = path.canonicalize() {
                 return this.handle_err(err, req).await;
             }
@@ -163,6 +207,16 @@ impl Service<ServiceRequest> for FilesService {
                 match this.index {
                     Some(ref index) => {
                         let named_path = path.join(index);
+                        if this.try_compressed {
+                            if let Some((named_file, encoding)) =
+                                find_compressed(&req, &named_path).await
+                            {
+                                return Ok(
+                                    this.serve_named_file_with_encoding(req, named_file, encoding)
+                                );
+                            }
+                        }
+                        // fallback to the uncompressed version
                         match NamedFile::open_async(named_path).await {
                             Ok(named_file) => Ok(this.serve_named_file(req, named_file)),
                             Err(_) if this.show_index => Ok(this.show_index(req, path)),
@@ -184,3 +238,84 @@ impl Service<ServiceRequest> for FilesService {
         })
     }
 }
+
+/// Flate doesn't have an accepted file extension, so it is not included here.
+const SUPPORTED_PRECOMPRESSION_ENCODINGS: &[header::ContentEncoding] = &[
+    header::ContentEncoding::Brotli,
+    header::ContentEncoding::Gzip,
+    header::ContentEncoding::Zstd,
+    header::ContentEncoding::Identity,
+];
+
+/// Searches disk for an acceptable alternate encoding of the content at the given path, as
+/// preferred by the request's `Accept-Encoding` header. Returns the corresponding `NamedFile` with
+/// the most appropriate supported encoding, if any exist.
+async fn find_compressed(
+    req: &ServiceRequest,
+    original_path: &Path,
+) -> Option<(NamedFile, header::ContentEncoding)> {
+    use actix_web::HttpMessage;
+    use header::{AcceptEncoding, ContentEncoding, Encoding};
+
+    // Retrieve the content type and content disposition based on the original filename. If we
+    // can't get these successfully, don't even try to find a compressed file.
+    let (content_type, content_disposition) =
+        match crate::named::get_content_type_and_disposition(original_path) {
+            Ok(values) => values,
+            Err(_) => return None,
+        };
+
+    let accept_encoding = req.get_header::<AcceptEncoding>()?;
+
+    let mut supported = SUPPORTED_PRECOMPRESSION_ENCODINGS
+        .iter()
+        .copied()
+        .map(Encoding::Known)
+        .collect::<Vec<_>>();
+
+    // Only move the original content-type/disposition into the chosen compressed file once.
+    let mut content_type = Some(content_type);
+    let mut content_disposition = Some(content_disposition);
+
+    loop {
+        // Select next acceptable encoding (honouring q=0 rejections) from remaining supported set.
+        let chosen = accept_encoding.negotiate(supported.iter())?;
+
+        let encoding = match chosen {
+            Encoding::Known(enc) => enc,
+            // No supported encoding should ever be unknown here.
+            Encoding::Unknown(_) => return None,
+        };
+
+        // Identity indicates there is no acceptable pre-compressed representation.
+        if encoding == ContentEncoding::Identity {
+            return None;
+        }
+
+        let extension = match encoding {
+            ContentEncoding::Brotli => ".br",
+            ContentEncoding::Gzip => ".gz",
+            ContentEncoding::Zstd => ".zst",
+            ContentEncoding::Identity => unreachable!(),
+            // Only variants in SUPPORTED_PRECOMPRESSION_ENCODINGS can occur here.
+            _ => unreachable!(),
+        };
+
+        let mut compressed_path = original_path.to_owned();
+        let mut filename = compressed_path.file_name()?.to_owned();
+        filename.push(extension);
+        compressed_path.set_file_name(filename);
+
+        match NamedFile::open_async(&compressed_path).await {
+            Ok(mut named_file) => {
+                named_file.content_type = content_type.take().unwrap();
+                named_file.content_disposition = content_disposition.take().unwrap();
+                return Some((named_file, encoding));
+            }
+            // Ignore errors while searching disk for a suitable encoding.
+            Err(_) => {
+                supported.retain(|enc| enc != &chosen);
+            }
+        }
+    }
+}

actix-files/tests/encoding.rs

@@ -36,6 +36,136 @@ async fn test_utf8_file_contents() {
     );
 }
 
+#[actix_web::test]
+async fn test_compression_encodings() {
+    use actix_web::body::MessageBody;
+
+    let utf8_txt_len = std::fs::metadata("./tests/utf8.txt").unwrap().len();
+    let utf8_txt_br_len = std::fs::metadata("./tests/utf8.txt.br").unwrap().len();
+    let utf8_txt_gz_len = std::fs::metadata("./tests/utf8.txt.gz").unwrap().len();
+
+    let srv =
+        test::init_service(App::new().service(Files::new("/", "./tests").try_compressed())).await;
+
+    // Select the requested encoding when present
+    let mut req = TestRequest::with_uri("/utf8.txt").to_request();
+    req.headers_mut().insert(
+        header::ACCEPT_ENCODING,
+        header::HeaderValue::from_static("gzip"),
+    );
+    let res = test::call_service(&srv, req).await;
+
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(
+        res.headers().get(header::CONTENT_TYPE),
+        Some(&HeaderValue::from_static("text/plain; charset=utf-8")),
+    );
+    assert_eq!(
+        res.headers().get(header::CONTENT_ENCODING),
+        Some(&HeaderValue::from_static("gzip")),
+    );
+    assert_eq!(
+        res.headers().get(header::VARY),
+        Some(&HeaderValue::from_static("accept-encoding")),
+    );
+    assert_eq!(
+        res.into_body().size(),
+        actix_web::body::BodySize::Sized(utf8_txt_gz_len),
+    );
+
+    // Select the highest priority encoding
+    let mut req = TestRequest::with_uri("/utf8.txt").to_request();
+    req.headers_mut().insert(
+        header::ACCEPT_ENCODING,
+        header::HeaderValue::from_static("gzip;q=0.6,br;q=0.8,*"),
+    );
+    let res = test::call_service(&srv, req).await;
+
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(
+        res.headers().get(header::CONTENT_TYPE),
+        Some(&HeaderValue::from_static("text/plain; charset=utf-8")),
+    );
+    assert_eq!(
+        res.headers().get(header::CONTENT_ENCODING),
+        Some(&HeaderValue::from_static("br")),
+    );
+    assert_eq!(
+        res.headers().get(header::VARY),
+        Some(&HeaderValue::from_static("accept-encoding")),
+    );
+    assert_eq!(
+        res.into_body().size(),
+        actix_web::body::BodySize::Sized(utf8_txt_br_len),
+    );
+
+    // Request encoding that doesn't exist on disk and fallback to no encoding
+    let mut req = TestRequest::with_uri("/utf8.txt").to_request();
+    req.headers_mut().insert(
+        header::ACCEPT_ENCODING,
+        header::HeaderValue::from_static("zstd"),
+    );
+    let res = test::call_service(&srv, req).await;
+
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(
+        res.headers().get(header::CONTENT_TYPE),
+        Some(&HeaderValue::from_static("text/plain; charset=utf-8")),
+    );
+    assert_eq!(res.headers().get(header::CONTENT_ENCODING), None,);
+    assert_eq!(
+        res.into_body().size(),
+        actix_web::body::BodySize::Sized(utf8_txt_len),
+    );
+
+    // Do not select an encoding explicitly refused via q=0
+    let mut req = TestRequest::with_uri("/utf8.txt").to_request();
+    req.headers_mut().insert(
+        header::ACCEPT_ENCODING,
+        header::HeaderValue::from_static("zstd;q=1, gzip;q=0"),
+    );
+    let res = test::call_service(&srv, req).await;
+
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(
+        res.headers().get(header::CONTENT_TYPE),
+        Some(&HeaderValue::from_static("text/plain; charset=utf-8")),
+    );
+    assert_eq!(res.headers().get(header::CONTENT_ENCODING), None,);
+    assert_eq!(
+        res.into_body().size(),
+        actix_web::body::BodySize::Sized(utf8_txt_len),
+    );
+
+    // Can still request a compressed file directly
+    let req = TestRequest::with_uri("/utf8.txt.gz").to_request();
+    let res = test::call_service(&srv, req).await;
+
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(
+        res.headers().get(header::CONTENT_TYPE),
+        Some(&HeaderValue::from_static("application/gzip")),
+    );
+    assert_eq!(res.headers().get(header::CONTENT_ENCODING), None,);
+
+    // Don't try compressed files
+    let srv = test::init_service(App::new().service(Files::new("/", "./tests"))).await;
+
+    let mut req = TestRequest::with_uri("/utf8.txt").to_request();
+    req.headers_mut().insert(
+        header::ACCEPT_ENCODING,
+        header::HeaderValue::from_static("gzip"),
+    );
+    let res = test::call_service(&srv, req).await;
+
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(
+        res.headers().get(header::CONTENT_TYPE),
+        Some(&HeaderValue::from_static("text/plain; charset=utf-8")),
+    );
+    assert_eq!(res.headers().get(header::CONTENT_ENCODING), None);
+}
+
 #[actix_web::test]
 async fn partial_range_response_encoding() {
     let srv = test::init_service(App::new().default_service(web::to(|| async {