From 42592e472a67e5ae5f399cb9ebc8abdf5e22a0cb Mon Sep 17 00:00:00 2001
From: Guillaume Friloux <guillaume.friloux@gmail.com>
Date: Mon, 10 Feb 2014 16:01:40 +0100
Subject: Fix an infinite loop if elasticsearch goes crazy.

We had a case here where elasticsearch stopped answering to HTTP queries
SMMAN was sending.

This led SMMAN to a state where he had 1015 sockets opened to
elasticsearch, and constantly trying to open logfiles as they had new
logs in it, but unable to open them because we reached the 1024 nofile
limit.

So this patch change behavior :
- Add a 10s timeout to ecore_con_url objects
- Dont instantly stat() files right after processing if an error
  happenned at last try.

This bug happenned twice here with a server sending over 1k logs in the
same second.
---
 src/lib/spy/spy_file.c     | 8 +++++++-
 src/lib/spy/spy_private.h  | 1 +
 src/lib/store/store_main.c | 1 +
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/lib/spy/spy_file.c b/src/lib/spy/spy_file.c
index 646115a..765f162 100644
--- a/src/lib/spy/spy_file.c
+++ b/src/lib/spy/spy_file.c
@@ -136,6 +136,7 @@ _spy_file_cb(void *data,
    if (sf->read.fd == -1)
      {
         ERR("Failed to open %s : %s", sf->name, strerror(errno));
+        sf->read.error = EINA_TRUE;
         return;
      }
 
@@ -151,6 +152,7 @@ _spy_file_cb(void *data,
         close(sf->read.fd);
         free(sf->read.databuf);
         sf->read.databuf = NULL;
+        sf->read.error = EINA_TRUE;
         return;
      }
 
@@ -183,7 +185,10 @@ _spy_file_end_cb(void *data,
 
    sf->poll.running = EINA_FALSE;
 
-   ecore_job_add(_spy_file_job, sf);
+   if (!sf->read.error)
+     ecore_job_add(_spy_file_job, sf);
+   else
+     sf->read.error = EINA_FALSE;
 }
 
 /**
@@ -254,6 +259,7 @@ spy_file_poll(void *data)
    sf->read.length = toread;
 
    sf->poll.running = EINA_TRUE;
+   sf->read.error = EINA_FALSE;
    et = ecore_thread_run(_spy_file_cb,
                          _spy_file_end_cb,
                          _spy_file_cancel_cb,
diff --git a/src/lib/spy/spy_private.h b/src/lib/spy/spy_private.h
index 4851bc9..3258546 100644
--- a/src/lib/spy/spy_private.h
+++ b/src/lib/spy/spy_private.h
@@ -37,6 +37,7 @@ struct _Spy_File
       Eina_Strbuf *buf;
       char *databuf;
       ssize_t nbr;
+      Eina_Bool error : 1;
    } read;
 
    struct
diff --git a/src/lib/store/store_main.c b/src/lib/store/store_main.c
index f1c964f..4f35cfc 100644
--- a/src/lib/store/store_main.c
+++ b/src/lib/store/store_main.c
@@ -95,6 +95,7 @@ store_add(Store *store,
    sa->data.sent = strdup(buf);
    DBG("store[%p] sa[%p] buf[%s]", store, sa, buf);
    ecore_con_url_data_set(sa->ec, sa);
+   ecore_con_url_timeout_set(sa->ec, 10.0);
    r = ecore_con_url_post(sa->ec, buf, len, "text/json");
    if (!r)
      {
-- 
cgit v1.2.1