Hard kill container on both a timeout or connection error

tractor_typed_msg_hackin
Tyler Goodlet 2022-07-06 10:13:27 -04:00
parent 7ec95ec00a
commit da54c80cac
1 changed files with 21 additions and 17 deletions

View File

@ -37,6 +37,7 @@ from docker.models.containers import Container as DockerContainer
from docker.errors import ( from docker.errors import (
DockerException, DockerException,
APIError, APIError,
ContainerError,
) )
from requests.exceptions import ConnectionError, ReadTimeout from requests.exceptions import ConnectionError, ReadTimeout
@ -96,9 +97,9 @@ async def open_docker(
# not perms? # not perms?
raise raise
finally: # finally:
if client: # if client:
client.close() # client.close()
class Container: class Container:
@ -185,6 +186,21 @@ class Container:
if 'is not running' in err.explanation: if 'is not running' in err.explanation:
return False return False
def hard_kill(self, start: float) -> None:
delay = time.time() - start
log.error(
f'Failed to kill container {cid} after {delay}s\n'
'sending SIGKILL..'
)
# get out the big guns, bc apparently marketstore
# doesn't actually know how to terminate gracefully
# :eyeroll:...
self.try_signal('SIGKILL')
self.cntr.wait(
timeout=3,
condition='not-running',
)
async def cancel( async def cancel(
self, self,
stop_msg: str, stop_msg: str,
@ -231,21 +247,9 @@ class Container:
ConnectionError, ConnectionError,
): ):
log.exception('Docker connection failure') log.exception('Docker connection failure')
break self.hard_kill(start)
else: else:
delay = time.time() - start self.hard_kill(start)
log.error(
f'Failed to kill container {cid} after {delay}s\n'
'sending SIGKILL..'
)
# get out the big guns, bc apparently marketstore
# doesn't actually know how to terminate gracefully
# :eyeroll:...
self.try_signal('SIGKILL')
self.cntr.wait(
timeout=3,
condition='not-running',
)
log.cancel(f'Container stopped: {cid}') log.cancel(f'Container stopped: {cid}')