serialhdl: Fail pending ack waits on mcu communication timeout

A command waiting for an ack from an mcu that stopped responding never
completes and may hold the gcode mutex. On a communication timeout fail
all current and future raw_send_wait_ack() calls with an error instead.

Signed-off-by: Maksim Bolgov <maksim8024@gmail.com>
Signed-off-by: Kevin O'Connor <kevin@koconnor.net>

klippy/mcu.py

@@ -905,6 +905,7 @@ class MCUConnectHelper:
                      self._name, eventtime)
         self._printer.invoke_shutdown("Lost communication with MCU '%s'" % (
             self._name,))
+        self._serial.abort_wait_acks()
     def is_shutdown(self):
         return self._is_shutdown
     def get_shutdown_msg(self):

klippy/serialhdl.py

@@ -38,6 +38,7 @@ class SerialReader:
         # Sent message notification tracking
         self.last_notify_id = 0
         self.pending_notifications = {}
+        self.can_wait_acks = True
     def _bg_thread(self):
         name_short = ("serialhdl %s" % (self.mcu_name))[:15]
         self.ffi_lib.set_thread_name(name_short.encode('utf-8'))
@@ -214,6 +215,12 @@ class SerialReader:
     def set_clock_est(self, freq, conv_time, conv_clock):
         self.ffi_lib.serialqueue_set_clock_est(
             self.serialqueue, freq, conv_time, conv_clock)
+    def abort_wait_acks(self):
+        self.can_wait_acks = False
+        # Background thread may be running, so work on copy
+        pending_notifications = self.pending_notifications.copy()
+        for pn in pending_notifications.values():
+            pn.complete(None)
     def disconnect(self):
         if self.serialqueue is not None:
             self.ffi_lib.serialqueue_exit(self.serialqueue)
@@ -223,9 +230,10 @@ class SerialReader:
         if self.serial_dev is not None:
             self.serial_dev.close()
             self.serial_dev = None
-        for pn in self.pending_notifications.values():
-            pn.complete(None)
+        # Abort any waiting commands and then restore ability to wait
+        self.abort_wait_acks()
         self.pending_notifications.clear()
+        self.can_wait_acks = True
     def stats(self, eventtime):
         if self.serialqueue is None:
             return ""
@@ -252,6 +260,8 @@ class SerialReader:
         self.ffi_lib.serialqueue_send(self.serialqueue, cmd_queue,
                                       cmd, len(cmd), minclock, reqclock, 0)
     def raw_send_wait_ack(self, cmd, minclock, reqclock, cmd_queue):
+        if not self.can_wait_acks:
+            self._error("No communication with MCU")
         self.last_notify_id += 1
         nid = self.last_notify_id
         completion = self.reactor.completion()
@@ -260,7 +270,7 @@ class SerialReader:
                                       cmd, len(cmd), minclock, reqclock, nid)
         params = completion.wait()
         if params is None:
-            self._error("Serial connection closed")
+            self._error("No communication with MCU")
         return params
     def send(self, msg, minclock=0, reqclock=0):
         cmd = self.msgparser.create_command(msg)