diff --git a/actix-server/src/accept.rs b/actix-server/src/accept.rs
index 2ed9d91c..771245fb 100644
--- a/actix-server/src/accept.rs
+++ b/actix-server/src/accept.rs
@@ -121,9 +121,7 @@ impl Availability {
         } else {
             let shift = 1 << idx as u128;
 
-            if self.0[offset] & shift != 0 {
-                self.0[offset] ^= shift;
-            }
+            self.0[offset] &= ~shift
         }
     }