watchdog - add a watchdog thread that pings mainloop every 10sec

if the main loop were to hang for some reason, this would detect it,
forcibly exit e and e_start can restart e again so things are working.

@feat

TODO

@@ -181,8 +181,6 @@ TODO:
    logging like tables, icons, timelines and graphs that can be output
    in text emulation and to screen - change eina log to go into here
    with eina_log_print_cb_set() )
-* watchdog: add watchdog handling to e_start to detect a hung e
-  * also detect if frames stop rendering but loop ok?
 * settings: config dialog redo and simplification
   * simplify the config and remove useless options
     * focus on new simpler config dialog(s) first

src/bin/e_includes.h

@@ -153,6 +153,7 @@
 #include "e_hints.h"
 #include "e_comp_x_devices.h"
 #include "e_comp_x_randr.h"
+#include "e_watchdog.h"
 
 #ifdef HAVE_WAYLAND
 # include "e_comp_wl.h"

src/bin/e_main.c

@@ -1086,6 +1086,7 @@ main(int argc, char **argv)
         E_LIST_FOREACH(e_comp->zones, e_comp_canvas_zone_restarted);
      }
 
+   e_watchdog_begin();
    TS("MAIN LOOP AT LAST");
    if (!setjmp(x_fatal_buff))
      {
@@ -1094,6 +1095,7 @@ main(int argc, char **argv)
      }
    else
      CRI("FATAL: X Died. Connection gone. Abbreviated Shutdown\n");
+   e_watchdog_end();
    e_main_loop_running = EINA_FALSE;
 
    inloop = EINA_FALSE;

src/bin/e_start_main.c

@@ -861,6 +861,12 @@ not_done:
                   restart = EINA_TRUE;
                   done = EINA_TRUE;
                }
+             else if (WEXITSTATUS(status) == 121)
+               {
+                  putenv("E_RESTART_OK=1");
+                  restart = EINA_TRUE;
+                  done = EINA_TRUE;
+               }
              else if (WEXITSTATUS(status) == 111)
                {
                   putenv("E_RESTART_OK=1");

src/bin/e_watchdog.c

@@ -0,0 +1,86 @@
+#include "e.h"
+
+static Ecore_Thread *_watchdog_thread = NULL;
+static Ecore_Pipe *_watchdog_pipe = NULL;
+static unsigned long last_seq = 0;
+
+static void
+_cb_watchdog_thread_pingpong_pipe(void *data EINA_UNUSED, void *buf, unsigned int bytes)
+{
+   unsigned long long *seq = buf;
+   unsigned long long seq_num = bytes / sizeof(int);
+
+   if (seq_num < 1) return; // XXX: error
+   last_seq = seq[seq_num - 1];
+}
+
+static void
+_cb_watchdog_thread_pingpong(void *data EINA_UNUSED, Ecore_Thread *thread)
+{
+   unsigned long long *seq_new;
+   unsigned long long seq = 0;
+
+   while (!ecore_thread_check(thread))
+     {
+        // send ping
+        seq_new = malloc(sizeof(unsigned long));
+        if (seq_new)
+          {
+             seq++;
+             *seq_new = seq;
+             ecore_thread_feedback(thread, seq_new);
+             // wait for ping from mainloop upto 10 sec
+             if (ecore_pipe_wait(_watchdog_pipe, 1, 10.0) < 1)
+               {
+                  printf("WD: Enlightenment main loop hung. No response to ping for 10sec\n");
+                  // do hard-exit as cleanup isnt doable
+                  _exit(121);
+               }
+             // wait another 10 sec before pinging
+             sleep(10);
+          }
+        else
+          {
+             printf("WD: Watchdog response alloc fail!!!!\n");
+             // XXX: alloc fail
+             break;
+          }
+     }
+}
+
+static void
+_cb_watchdog_thread_pingpong_reply(void *data EINA_UNUSED, Ecore_Thread *thread EINA_UNUSED, void *msg)
+{
+   // repluy back to mainloop with same ping number
+   unsigned long long *seq = msg;
+   ecore_pipe_write(_watchdog_pipe, seq, sizeof(unsigned long long));
+   free(seq);
+}
+
+static void
+_cb_watchdog_thread_pingpong_end(void *data EINA_UNUSED, Ecore_Thread *thread EINA_UNUSED)
+{
+   ecore_pipe_del(_watchdog_pipe);
+   _watchdog_pipe = NULL;
+   _watchdog_thread = NULL;
+}
+
+E_API void
+e_watchdog_begin(void)
+{
+   // set up main-loop ping-pong to a thread
+   _watchdog_pipe = ecore_pipe_add(_cb_watchdog_thread_pingpong_pipe, NULL);
+   _watchdog_thread = ecore_thread_feedback_run
+     (_cb_watchdog_thread_pingpong,
+      _cb_watchdog_thread_pingpong_reply,
+      _cb_watchdog_thread_pingpong_end,
+      NULL,
+      NULL, EINA_TRUE);
+}
+
+E_API void
+e_watchdog_end(void)
+{
+   if (_watchdog_thread) ecore_thread_cancel(_watchdog_thread);
+   _watchdog_thread = NULL;
+}

src/bin/e_watchdog.h

@@ -0,0 +1,10 @@
+#ifdef E_TYPEDEFS
+#else
+# ifndef E_WATCHDOG_H
+#  define E_WATCHDOG_H
+
+E_API void e_watchdog_begin(void);
+E_API void e_watchdog_end(void);
+
+# endif
+#endif

src/bin/meson.build

@@ -186,6 +186,7 @@ src = [
   'e_user.c',
   'e_utils.c',
   'e_video.c',
+  'e_watchdog.c',
   'e_widget_aspect.c',
   'e_widget_button.c',
   'e_widget.c',
@@ -363,6 +364,7 @@ hdr = [
   'e_user.h',
   'e_utils.h',
   'e_video.h',
+  'e_watchdog.h',
   'e_widget_aspect.h',
   'e_widget_button.h',
   'e_widget_check.h',