From f356fb0a68503b5695bce5b95d1b7cda8335043c Mon Sep 17 00:00:00 2001
From: Tyler Goodlet <jgbt@protonmail.com>
Date: Wed, 6 Jul 2022 10:13:27 -0400
Subject: [PATCH 1/3] Hard kill container on both a timeout or connection error

---
 piker/data/_ahab.py | 38 +++++++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/piker/data/_ahab.py b/piker/data/_ahab.py
index 98b3b41f..b4802cf1 100644
--- a/piker/data/_ahab.py
+++ b/piker/data/_ahab.py
@@ -37,6 +37,7 @@ from docker.models.containers import Container as DockerContainer
 from docker.errors import (
     DockerException,
     APIError,
+    ContainerError,
 )
 from requests.exceptions import ConnectionError, ReadTimeout
 
@@ -96,9 +97,9 @@ async def open_docker(
         # not perms?
         raise
 
-    finally:
-        if client:
-            client.close()
+    # finally:
+    #     if client:
+    #         client.close()
 
 
 class Container:
@@ -185,6 +186,21 @@ class Container:
             if 'is not running' in err.explanation:
                 return False
 
+    def hard_kill(self, start: float) -> None:
+        delay = time.time() - start
+        log.error(
+            f'Failed to kill container {cid} after {delay}s\n'
+            'sending SIGKILL..'
+        )
+        # get out the big guns, bc apparently marketstore
+        # doesn't actually know how to terminate gracefully
+        # :eyeroll:...
+        self.try_signal('SIGKILL')
+        self.cntr.wait(
+            timeout=3,
+            condition='not-running',
+        )
+
     async def cancel(
         self,
         stop_msg: str,
@@ -231,21 +247,9 @@ class Container:
                 ConnectionError,
             ):
                 log.exception('Docker connection failure')
-                break
+                self.hard_kill(start)
         else:
-            delay = time.time() - start
-            log.error(
-                f'Failed to kill container {cid} after {delay}s\n'
-                'sending SIGKILL..'
-            )
-            # get out the big guns, bc apparently marketstore
-            # doesn't actually know how to terminate gracefully
-            # :eyeroll:...
-            self.try_signal('SIGKILL')
-            self.cntr.wait(
-                timeout=3,
-                condition='not-running',
-            )
+            self.hard_kill(start)
 
         log.cancel(f'Container stopped: {cid}')
 

From 019867b413bfc1a1fdd44cf6a061b838115ab4aa Mon Sep 17 00:00:00 2001
From: Tyler Goodlet <jgbt@protonmail.com>
Date: Thu, 7 Jul 2022 17:10:06 -0400
Subject: [PATCH 2/3] Fix missing container id, drop custom exception

---
 piker/data/_ahab.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/piker/data/_ahab.py b/piker/data/_ahab.py
index b4802cf1..d04d73a2 100644
--- a/piker/data/_ahab.py
+++ b/piker/data/_ahab.py
@@ -51,10 +51,6 @@ class DockerNotStarted(Exception):
     'Prolly you dint start da daemon bruh'
 
 
-class ContainerError(RuntimeError):
-    'Error reported via app-container logging level'
-
-
 @acm
 async def open_docker(
     url: Optional[str] = None,
@@ -189,7 +185,7 @@ class Container:
     def hard_kill(self, start: float) -> None:
         delay = time.time() - start
         log.error(
-            f'Failed to kill container {cid} after {delay}s\n'
+            f'Failed to kill container {self.cntr.id} after {delay}s\n'
             'sending SIGKILL..'
         )
         # get out the big guns, bc apparently marketstore

From 4c7c78c815a08f6c9dc96a229b70a1e05e99b1b4 Mon Sep 17 00:00:00 2001
From: Tyler Goodlet <jgbt@protonmail.com>
Date: Fri, 8 Jul 2022 17:18:05 -0400
Subject: [PATCH 3/3] Add a `ApplicationLogError` custom exc instead

---
 piker/data/_ahab.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/piker/data/_ahab.py b/piker/data/_ahab.py
index d04d73a2..45203447 100644
--- a/piker/data/_ahab.py
+++ b/piker/data/_ahab.py
@@ -37,7 +37,7 @@ from docker.models.containers import Container as DockerContainer
 from docker.errors import (
     DockerException,
     APIError,
-    ContainerError,
+    # ContainerError,
 )
 from requests.exceptions import ConnectionError, ReadTimeout
 
@@ -51,6 +51,10 @@ class DockerNotStarted(Exception):
     'Prolly you dint start da daemon bruh'
 
 
+class ApplicationLogError(Exception):
+    'App in container reported an error in logs'
+
+
 @acm
 async def open_docker(
     url: Optional[str] = None,
@@ -153,7 +157,7 @@ class Container:
 
                     # print(f'level: {level}')
                     if level in ('error', 'fatal'):
-                        raise ContainerError(msg)
+                        raise ApplicationLogError(msg)
 
                 if patt in msg:
                     return True