Hard kill container on both a timeout or connection error
parent
7ec95ec00a
commit
da54c80cac
|
@ -37,6 +37,7 @@ from docker.models.containers import Container as DockerContainer
|
||||||
from docker.errors import (
|
from docker.errors import (
|
||||||
DockerException,
|
DockerException,
|
||||||
APIError,
|
APIError,
|
||||||
|
ContainerError,
|
||||||
)
|
)
|
||||||
from requests.exceptions import ConnectionError, ReadTimeout
|
from requests.exceptions import ConnectionError, ReadTimeout
|
||||||
|
|
||||||
|
@ -96,9 +97,9 @@ async def open_docker(
|
||||||
# not perms?
|
# not perms?
|
||||||
raise
|
raise
|
||||||
|
|
||||||
finally:
|
# finally:
|
||||||
if client:
|
# if client:
|
||||||
client.close()
|
# client.close()
|
||||||
|
|
||||||
|
|
||||||
class Container:
|
class Container:
|
||||||
|
@ -185,6 +186,21 @@ class Container:
|
||||||
if 'is not running' in err.explanation:
|
if 'is not running' in err.explanation:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def hard_kill(self, start: float) -> None:
|
||||||
|
delay = time.time() - start
|
||||||
|
log.error(
|
||||||
|
f'Failed to kill container {cid} after {delay}s\n'
|
||||||
|
'sending SIGKILL..'
|
||||||
|
)
|
||||||
|
# get out the big guns, bc apparently marketstore
|
||||||
|
# doesn't actually know how to terminate gracefully
|
||||||
|
# :eyeroll:...
|
||||||
|
self.try_signal('SIGKILL')
|
||||||
|
self.cntr.wait(
|
||||||
|
timeout=3,
|
||||||
|
condition='not-running',
|
||||||
|
)
|
||||||
|
|
||||||
async def cancel(
|
async def cancel(
|
||||||
self,
|
self,
|
||||||
stop_msg: str,
|
stop_msg: str,
|
||||||
|
@ -231,21 +247,9 @@ class Container:
|
||||||
ConnectionError,
|
ConnectionError,
|
||||||
):
|
):
|
||||||
log.exception('Docker connection failure')
|
log.exception('Docker connection failure')
|
||||||
break
|
self.hard_kill(start)
|
||||||
else:
|
else:
|
||||||
delay = time.time() - start
|
self.hard_kill(start)
|
||||||
log.error(
|
|
||||||
f'Failed to kill container {cid} after {delay}s\n'
|
|
||||||
'sending SIGKILL..'
|
|
||||||
)
|
|
||||||
# get out the big guns, bc apparently marketstore
|
|
||||||
# doesn't actually know how to terminate gracefully
|
|
||||||
# :eyeroll:...
|
|
||||||
self.try_signal('SIGKILL')
|
|
||||||
self.cntr.wait(
|
|
||||||
timeout=3,
|
|
||||||
condition='not-running',
|
|
||||||
)
|
|
||||||
|
|
||||||
log.cancel(f'Container stopped: {cid}')
|
log.cancel(f'Container stopped: {cid}')
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue