diff --git a/actix-server/src/accept.rs b/actix-server/src/accept.rs
index 026bcc37..2ed9d91c 100644
--- a/actix-server/src/accept.rs
+++ b/actix-server/src/accept.rs
@@ -121,9 +121,9 @@ impl Availability {
         } else {
             let shift = 1 << idx as u128;
 
-            debug_assert_ne!(self.0[offset] & shift, 0);
-
-            self.0[offset] ^= shift;
+            if self.0[offset] & shift != 0 {
+                self.0[offset] ^= shift;
+            }
         }
     }
 
@@ -527,6 +527,9 @@ mod test {
 
         aval.set_available(idx, false);
         assert!(!aval.available());
+
+        aval.set_available(idx, false);
+        assert!(!aval.available());
     }
 
     fn multi(aval: &mut Availability, mut idx: Vec<usize>) {
@@ -565,13 +568,6 @@ mod test {
         single(&mut aval, 512);
     }
 
-    #[test]
-    #[should_panic]
-    fn double_set_unavailable() {
-        let mut aval = Availability::default();
-        aval.set_available(233, false);
-    }
-
     #[test]
     fn pin_point() {
         let mut aval = Availability::default();