feat(theme): restructure automatic color selection

BREAKING CHANGES:
* the default theme now prefers ANSI colors, even if RGB is supported
* `MietteHandlerOpts::ansi_colors` is removed
* `MietteHandlerOpts::rgb_color` now takes an enum that controls the
  color format used when color support is enabled, and has no effect
  otherwise.

Cargo.toml

@@ -35,6 +35,8 @@ indenter = "0.3.0"
 rustversion = "1.0"
 trybuild = { version = "1.0.19", features = ["diff"] }
 syn = { version = "1.0", features = ["full"] }
+regex = "1.5"
+lazy_static = "1.4"
 
 [features]
 default = []

src/handler.rs

@@ -10,6 +10,23 @@ use crate::ReportHandler;
 use crate::ThemeCharacters;
 use crate::ThemeStyles;
 
+/// Settings to control the color format used for graphical rendering.
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub enum RgbColors {
+    /// Use RGB colors even if the terminal does not support them
+    Always,
+    /// Use RGB colors instead of ANSI if the terminal supports RGB
+    Preferred,
+    /// Always use ANSI, regardless of terminal support for RGB
+    Never,
+}
+
+impl Default for RgbColors {
+    fn default() -> RgbColors {
+        RgbColors::Never
+    }
+}
+
 /**
 Create a custom [`MietteHandler`] from options.
 
@@ -33,8 +50,7 @@ pub struct MietteHandlerOpts {
     pub(crate) theme: Option<GraphicalTheme>,
     pub(crate) force_graphical: Option<bool>,
     pub(crate) force_narrated: Option<bool>,
-    pub(crate) ansi_colors: Option<bool>,
+    pub(crate) rgb_colors: RgbColors,
-    pub(crate) rgb_colors: Option<bool>,
     pub(crate) color: Option<bool>,
     pub(crate) unicode: Option<bool>,
     pub(crate) footer: Option<String>,
@@ -71,16 +87,31 @@ impl MietteHandlerOpts {
         self
     }
 
-    /// If true, colors will be used during graphical rendering. Actual color
+    /// If true, colors will be used during graphical rendering, regardless
-    /// format will be auto-detected.
+    /// of whether or not the terminal supports them.
+    ///
+    /// If false, colors will never be used.
+    ///
+    /// If unspecified, colors will be used only if the terminal supports them.
+    ///
+    /// The actual format depends on the value of
+    /// [`MietteHandlerOpts::rgb_colors`].
     pub fn color(mut self, color: bool) -> Self {
         self.color = Some(color);
         self
     }
 
-    /// If true, RGB colors will be used during graphical rendering.
+    /// Controls which color format to use if colors are used in graphical
-    pub fn rgb_colors(mut self, color: bool) -> Self {
+    /// rendering.
-        self.rgb_colors = Some(color);
+    ///
+    /// The default is `Never`.
+    ///
+    /// This value does not control whether or not colors are being used in the
+    /// first place. That is handled by the [`MietteHandlerOpts::color`]
+    /// setting. If colors are not being used, the value of `rgb_colors` has
+    /// no effect.
+    pub fn rgb_colors(mut self, color: RgbColors) -> Self {
+        self.rgb_colors = color;
         self
     }
 
@@ -91,11 +122,6 @@ impl MietteHandlerOpts {
         self
     }
 
-    /// If true, ANSI colors will be used during graphical rendering.
-    pub fn ansi_colors(mut self, color: bool) -> Self {
-        self.ansi_colors = Some(color);
-        self
-    }
     /// If true, graphical rendering will be used regardless of terminal
     /// detection.
     pub fn force_graphical(mut self, force: bool) -> Self {
@@ -152,18 +178,17 @@ impl MietteHandlerOpts {
             };
             let styles = if self.color == Some(false) {
                 ThemeStyles::none()
-            } else if self.rgb_colors == Some(true) {
+            } else if let Some(color) = supports_color::on(Stream::Stderr) {
-                ThemeStyles::rgb()
+                match self.rgb_colors {
-            } else if self.ansi_colors == Some(true) {
+                    RgbColors::Always => ThemeStyles::rgb(),
-                ThemeStyles::ansi()
+                    RgbColors::Preferred if color.has_16m => ThemeStyles::rgb(),
-            } else if let Some(colors) = supports_color::on(Stream::Stderr) {
+                    _ => ThemeStyles::ansi(),
-                if colors.has_16m {
-                    ThemeStyles::rgb()
-                } else {
-                    ThemeStyles::ansi()
                 }
             } else if self.color == Some(true) {
-                ThemeStyles::ansi()
+                match self.rgb_colors {
+                    RgbColors::Always => ThemeStyles::rgb(),
+                    _ => ThemeStyles::ansi(),
+                }
             } else {
                 ThemeStyles::none()
             };

tests/color_format.rs

@@ -0,0 +1,137 @@
+#![cfg(feature = "fancy-no-backtrace")]
+
+use lazy_static::lazy_static;
+use miette::{Diagnostic, MietteHandler, MietteHandlerOpts, ReportHandler, RgbColors};
+use regex::Regex;
+use std::fmt::{self, Debug};
+use std::sync::Mutex;
+use thiserror::Error;
+
+#[derive(Eq, PartialEq, Debug)]
+enum ColorFormat {
+    NoColor,
+    Ansi,
+    Rgb,
+}
+
+#[derive(Debug, Diagnostic, Error)]
+#[error("oops!")]
+#[diagnostic(code(oops::my::bad), help("try doing it better next time?"))]
+struct MyBad;
+
+struct FormatTester(MietteHandler);
+
+impl Debug for FormatTester {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        self.0.debug(&MyBad, f)
+    }
+}
+
+/// Check the color format used by a handler.
+fn color_format(handler: MietteHandler) -> ColorFormat {
+    let out = format!("{:?}", FormatTester(handler));
+
+    let rgb_colors = Regex::new(r"\u{1b}\[[34]8;2;").unwrap();
+    let ansi_colors = Regex::new(r"\u{1b}\[(3|4|9|10)[0-7][m;]").unwrap();
+    if rgb_colors.is_match(&out) {
+        ColorFormat::Rgb
+    } else if ansi_colors.is_match(&out) {
+        ColorFormat::Ansi
+    } else {
+        ColorFormat::NoColor
+    }
+}
+
+/// Runs a function with an environment variable set to a specific value, then
+/// sets it back to it's original value once completed.
+fn with_env_var<F: FnOnce()>(var: &str, value: &str, body: F) {
+    let old_value = std::env::var_os(var);
+    std::env::set_var(var, value);
+    body();
+    if let Some(old_value) = old_value {
+        std::env::set_var(var, old_value);
+    } else {
+        std::env::remove_var(var);
+    }
+}
+
+lazy_static! {
+    static ref COLOR_ENV_VARS: Mutex<()> = Mutex::new(());
+}
+
+/// Assert the color format used by a handler with different levels of terminal
+/// support.
+fn check_colors<F: Fn(MietteHandlerOpts) -> MietteHandlerOpts>(
+    make_handler: F,
+    no_support: ColorFormat,
+    ansi_support: ColorFormat,
+    rgb_support: ColorFormat,
+) {
+    // To simulate different levels of terminal support we're using specific
+    // environment variables that are handled by the supports_color crate.
+    //
+    // Since environment variables are shared for the entire process, we need
+    // to ensure that only one test that modifies these env vars runs at a time.
+    let guard = COLOR_ENV_VARS.lock().unwrap();
+
+    with_env_var("NO_COLOR", "1", || {
+        let handler = make_handler(MietteHandlerOpts::new()).build();
+        assert_eq!(color_format(handler), no_support);
+    });
+    with_env_var("FORCE_COLOR", "1", || {
+        let handler = make_handler(MietteHandlerOpts::new()).build();
+        assert_eq!(color_format(handler), ansi_support);
+    });
+    with_env_var("FORCE_COLOR", "3", || {
+        let handler = make_handler(MietteHandlerOpts::new()).build();
+        assert_eq!(color_format(handler), rgb_support);
+    });
+
+    drop(guard);
+}
+
+#[test]
+fn no_color_preference() {
+    use ColorFormat::*;
+    check_colors(|opts| opts, NoColor, Ansi, Ansi);
+}
+
+#[test]
+fn color_never() {
+    use ColorFormat::*;
+    check_colors(|opts| opts.color(false), NoColor, NoColor, NoColor);
+}
+
+#[test]
+fn color_always() {
+    use ColorFormat::*;
+    check_colors(|opts| opts.color(true), Ansi, Ansi, Ansi);
+}
+
+#[test]
+fn rgb_preferred() {
+    use ColorFormat::*;
+    check_colors(
+        |opts| opts.rgb_colors(RgbColors::Preferred),
+        NoColor,
+        Ansi,
+        Rgb,
+    );
+}
+
+#[test]
+fn rgb_always() {
+    use ColorFormat::*;
+    check_colors(|opts| opts.rgb_colors(RgbColors::Always), NoColor, Rgb, Rgb);
+}
+
+#[test]
+fn color_always_rgb_always() {
+    use ColorFormat::*;
+    check_colors(
+        |opts| opts.color(true).rgb_colors(RgbColors::Always),
+        Rgb,
+        Rgb,
+        Rgb,
+    );
+}