diff --git a/src/handlers/graphical.rs b/src/handlers/graphical.rs
index e9e84b8..9bba215 100644
--- a/src/handlers/graphical.rs
+++ b/src/handlers/graphical.rs
@@ -446,10 +446,16 @@ impl GraphicalReportHandler {
     ) -> fmt::Result {
         let (contents, lines) = self.get_lines(source, context.inner())?;
 
-        let primary_label = labels
-            .iter()
+        // only consider labels from the context as primary label
+        let ctx_labels = labels.iter().filter(|l| {
+            context.inner().offset() <= l.inner().offset()
+                && l.inner().offset() + l.inner().len()
+                    <= context.inner().offset() + context.inner().len()
+        });
+        let primary_label = ctx_labels
+            .clone()
             .find(|label| label.primary())
-            .or_else(|| labels.first());
+            .or_else(|| ctx_labels.clone().next());
 
         // sorting is your friend
         let labels = labels