make server start panic free. All errors are caught and return as std::io::Error

This commit is contained in:
fakeshadow 2021-04-13 06:11:20 +08:00
parent 04102867dd
commit 42c53159da
3 changed files with 92 additions and 101 deletions

View File

@ -60,7 +60,7 @@ impl Accept {
pub(crate) fn start( pub(crate) fn start(
sockets: Vec<(Token, MioListener)>, sockets: Vec<(Token, MioListener)>,
builder: &ServerBuilder, builder: &ServerBuilder,
) -> io::Result<(WakerQueue, Vec<(usize, WorkerHandleServer)>)> { ) -> io::Result<(WakerQueue, Vec<WorkerHandleServer>)> {
let server_handle = ServerHandle::new(builder.cmd_tx.clone()); let server_handle = ServerHandle::new(builder.cmd_tx.clone());
// construct poll instance and it's waker // construct poll instance and it's waker
@ -73,9 +73,8 @@ impl Accept {
// start workers // start workers
let availability = WorkerAvailability::new(waker_queue.clone()); let availability = WorkerAvailability::new(waker_queue.clone());
let factories = builder.services.iter().map(|v| v.clone_factory()).collect(); let factories = builder.services.iter().map(|v| v.clone_factory()).collect();
let (handle_accept, handle_server) =
ServerWorker::start(idx, factories, availability, builder.worker_config)?; ServerWorker::start(idx, factories, availability, builder.worker_config)
Ok((handle_accept, (idx, handle_server)))
}) })
.collect::<Result<Vec<_>, io::Error>>()? .collect::<Result<Vec<_>, io::Error>>()?
.into_iter() .into_iter()

View File

@ -23,12 +23,16 @@ use crate::worker::{ServerWorker, ServerWorkerConfig, WorkerAvailability, Worker
/// When awaited or spawned would listen to signal and message from [ServerHandle](ServerHandle). /// When awaited or spawned would listen to signal and message from [ServerHandle](ServerHandle).
#[must_use = "futures do nothing unless you `.await` or poll them"] #[must_use = "futures do nothing unless you `.await` or poll them"]
pub struct Server { pub enum Server {
Server(ServerInner),
Error(Option<io::Error>),
}
pub struct ServerInner {
cmd_tx: UnboundedSender<ServerCommand>, cmd_tx: UnboundedSender<ServerCommand>,
cmd_rx: UnboundedReceiver<ServerCommand>, cmd_rx: UnboundedReceiver<ServerCommand>,
handles: Vec<(usize, WorkerHandleServer)>, handles: Vec<WorkerHandleServer>,
services: Vec<Box<dyn InternalServiceFactory>>, services: Vec<Box<dyn InternalServiceFactory>>,
notify: Vec<oneshot::Sender<()>>,
exit: bool, exit: bool,
worker_config: ServerWorkerConfig, worker_config: ServerWorkerConfig,
signals: Option<Signals>, signals: Option<Signals>,
@ -52,32 +56,33 @@ impl Server {
.collect(); .collect();
// start accept thread. return waker_queue and worker handles. // start accept thread. return waker_queue and worker handles.
let (waker_queue, handles) = Accept::start(sockets, &builder) match Accept::start(sockets, &builder) {
// TODO: include error to Server type and poll return it in Future. Ok((waker_queue, handles)) => {
.unwrap_or_else(|e| panic!("Can not start Accept: {}", e)); // construct signals future.
let signals = if !builder.no_signals {
// Check tokio runtime.
if tokio::runtime::Handle::try_current().is_err() {
let err = io::Error::new(io::ErrorKind::Other, "there is no reactor running. Please enable ServerBuilder::disable_signals when start server in non tokio 1.x runtime.");
return Self::Error(Some(err));
}
Some(Signals::new())
} else {
None
};
// construct signals future. Self::Server(ServerInner {
let signals = if !builder.no_signals { cmd_tx: builder.cmd_tx,
// Check tokio runtime. cmd_rx: builder.cmd_rx,
tokio::runtime::Handle::try_current() handles,
.map(|_|()) services: builder.services,
.expect("there is no reactor running. Please enable ServerBuilder::disable_signals when start server in non tokio 1.x runtime."); exit: builder.exit,
Some(Signals::new()) worker_config: builder.worker_config,
} else { signals,
None on_stop_task: None,
}; waker_queue,
})
Self { }
cmd_tx: builder.cmd_tx, Err(e) => Self::Error(Some(e)),
cmd_rx: builder.cmd_rx,
handles,
services: builder.services,
notify: Vec::new(),
exit: builder.exit,
worker_config: builder.worker_config,
signals,
on_stop_task: None,
waker_queue,
} }
} }
@ -85,9 +90,17 @@ impl Server {
/// ///
/// See [ServerHandle](ServerHandle) for usage. /// See [ServerHandle](ServerHandle) for usage.
pub fn handle(&self) -> ServerHandle { pub fn handle(&self) -> ServerHandle {
ServerHandle::new(self.cmd_tx.clone()) match self {
Self::Server(ref inner) => ServerHandle::new(inner.cmd_tx.clone()),
Self::Error(err) => panic!(
"ServerHandle can not be obtained. Server failed to start due to error: {:?}",
err.as_ref().unwrap()
),
}
} }
}
impl ServerInner {
fn handle_cmd(&mut self, item: ServerCommand) -> Option<BoxFuture<'static, ()>> { fn handle_cmd(&mut self, item: ServerCommand) -> Option<BoxFuture<'static, ()>> {
match item { match item {
ServerCommand::Pause(tx) => { ServerCommand::Pause(tx) => {
@ -139,14 +152,13 @@ impl Server {
// stop accept thread // stop accept thread
self.waker_queue.wake(WakerInterest::Stop); self.waker_queue.wake(WakerInterest::Stop);
let notify = std::mem::take(&mut self.notify);
// stop workers // stop workers
if !self.handles.is_empty() && graceful { if !self.handles.is_empty() && graceful {
let iter = self let iter = self
.handles .handles
.iter() .iter()
.map(move |worker| worker.1.stop(graceful)) .map(move |worker| worker.stop(graceful))
.collect::<Vec<_>>(); .collect::<Vec<_>>();
// TODO: this async block can return io::Error. // TODO: this async block can return io::Error.
@ -157,9 +169,6 @@ impl Server {
if let Some(tx) = completion { if let Some(tx) = completion {
let _ = tx.send(()); let _ = tx.send(());
} }
for tx in notify {
let _ = tx.send(());
}
if exit { if exit {
sleep(Duration::from_millis(300)).await; sleep(Duration::from_millis(300)).await;
System::try_current().as_ref().map(System::stop); System::try_current().as_ref().map(System::stop);
@ -176,52 +185,32 @@ impl Server {
if let Some(tx) = completion { if let Some(tx) = completion {
let _ = tx.send(()); let _ = tx.send(());
} }
for tx in notify {
let _ = tx.send(());
}
})) }))
} }
} }
ServerCommand::WorkerFaulted(idx) => { ServerCommand::WorkerFaulted(idx) => {
let mut found = false; assert!(self.handles.iter().any(|handle| handle.idx == idx));
for i in 0..self.handles.len() {
if self.handles[i].0 == idx { error!("Worker {} has died, restarting", idx);
self.handles.swap_remove(i);
found = true; let availability = WorkerAvailability::new(self.waker_queue.clone());
break; let factories = self
} .services
} .iter()
.map(|service| service.clone_factory())
if found { .collect();
error!("Worker has died {:?}, restarting", idx); let res = ServerWorker::start(idx, factories, availability, self.worker_config);
let mut new_idx = self.handles.len(); match res {
'found: loop { Ok((handle_accept, handle_server)) => {
for i in 0..self.handles.len() { *self
if self.handles[i].0 == new_idx { .handles
new_idx += 1; .iter_mut()
continue 'found; .find(|handle| handle.idx == idx)
} .unwrap() = handle_server;
} self.waker_queue.wake(WakerInterest::Worker(handle_accept));
break;
}
let availability = WorkerAvailability::new(self.waker_queue.clone());
let factories = self.services.iter().map(|v| v.clone_factory()).collect();
let res = ServerWorker::start(
new_idx,
factories,
availability,
self.worker_config,
);
match res {
Ok((handle_accept, handle_server)) => {
self.handles.push((new_idx, handle_server));
self.waker_queue.wake(WakerInterest::Worker(handle_accept));
}
Err(e) => error!("Can not start worker: {:?}", e),
} }
Err(e) => error!("Can not start worker: {:?}", e),
} }
None None
@ -234,28 +223,31 @@ impl Future for Server {
type Output = io::Result<()>; type Output = io::Result<()>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.as_mut().get_mut(); match self.as_mut().get_mut() {
Self::Error(e) => Poll::Ready(Err(e.take().unwrap())),
// poll signals first. remove it on resolve. Self::Server(inner) => {
if let Some(ref mut signals) = this.signals { // poll signals first. remove it on resolve.
if let Poll::Ready(signal) = Pin::new(signals).poll(cx) { if let Some(ref mut signals) = inner.signals {
this.on_stop_task = this.handle_cmd(ServerCommand::Signal(signal)); if let Poll::Ready(signal) = Pin::new(signals).poll(cx) {
this.signals = None; inner.on_stop_task = inner.handle_cmd(ServerCommand::Signal(signal));
} inner.signals = None;
} }
}
// actively poll command channel and handle command.
loop { // actively poll command channel and handle command.
// got on stop task. resolve it exclusively and exit. loop {
if let Some(ref mut fut) = this.on_stop_task { // got on stop task. resolve it exclusively and exit.
return fut.as_mut().poll(cx).map(|_| Ok(())); if let Some(ref mut fut) = inner.on_stop_task {
} return fut.as_mut().poll(cx).map(|_| Ok(()));
}
match Pin::new(&mut this.cmd_rx).poll_recv(cx) {
Poll::Ready(Some(it)) => { match Pin::new(&mut inner.cmd_rx).poll_recv(cx) {
this.on_stop_task = this.handle_cmd(it); Poll::Ready(Some(it)) => {
inner.on_stop_task = inner.handle_cmd(it);
}
_ => return Poll::Pending,
}
} }
_ => return Poll::Pending,
} }
} }
} }

View File

@ -200,10 +200,10 @@ impl ServerWorker {
config: ServerWorkerConfig, config: ServerWorkerConfig,
) -> io::Result<(WorkerHandleAccept, WorkerHandleServer)> { ) -> io::Result<(WorkerHandleAccept, WorkerHandleServer)> {
assert!(!availability.available()); assert!(!availability.available());
let avail = availability.clone();
let (tx1, rx) = unbounded_channel(); let (tx1, rx) = unbounded_channel();
let (tx2, rx2) = unbounded_channel(); let (tx2, rx2) = unbounded_channel();
let avail = availability.clone();
// Try to get actix system when have one. // Try to get actix system when have one.
let system = System::try_current(); let system = System::try_current();