Make Smman able to filter logs and tag them.

Next step is turning logs into JSON and send them to ES.

12 files changed, 228 insertions, 19 deletions

diff --git a/src/bin/Makefile.mk b/src/bin/Makefile.mk
index 6ab1a81..4547c5d 100644
--- a/src/bin/Makefile.mk
+++ b/src/bin/Makefile.mk
@@ -8,9 +8,11 @@ src_bin_smman_SOURCES = \
 src/bin/main.c \
 src/bin/config.c \
 src/bin/filter.c \
+src/bin/log.c \
 src/bin/smman.h
 src_bin_smman_CPPFLAGS = @BIN_CFLAGS@ $(EXTRA_CPPFLAGS)
 src_bin_smman_LDFLAGS = @BIN_LIBS@
 src_bin_smman_LDADD = \
 src/lib/libconf.la \
-src/lib/librules.la
+src/lib/librules.la \
+src/lib/libspy.la
diff --git a/src/bin/filter.c b/src/bin/filter.c
index ef05a49..274f0cb 100644
--- a/src/bin/filter.c
+++ b/src/bin/filter.c
@@ -1,4 +1,25 @@
 #include "smman.h"
+#include <glob.h>
+
+static void
+_filter_rules_free(void *data)
+{
+   rules_rule_free((Rule *)data);
+}
+
+Filter *
+_filter_find(Smman *smman,
+             const char *filename)
+{
+   Filter *filter;
+
+   EINA_INLIST_FOREACH(smman->filters, filter)
+     {
+        if (!strcmp(filter->filename, filename))
+          return filter;
+     }
+   return NULL;
+}
 
 void
 filter_load(void *data,
@@ -6,6 +27,11 @@ filter_load(void *data,
             Rule *rule)
 {
    Smman *smman;
+   Filter *filter;
+   int r;
+   glob_t files;
+   char **s;
+   size_t i;
 
    smman = data;
    if (smman->rules != rules)
@@ -18,6 +44,38 @@ filter_load(void *data,
        rule->name, rule->spec.filename, rule->spec.source_host,
        rule->spec.source_path, rule->spec.tags,
        (rule->spec.todel) ? "EINA_TRUE" : "EINA_FALSE");
+
+   /* Loop with globbing to write here! */
+   r = glob(rule->spec.filename, GLOB_MARK, 0, &files);
+   if (r)
+     {
+        ERR("Unable to find files matching \"%s\"", rule->spec.filename);
+        return;
+     }
+
+   for (s = files.gl_pathv, i = files.gl_pathc; i; s++, i--)
+     {
+        DBG("Corresponding file %s", *s);
+
+        /* We have to check the files affected by this rule */
+        filter = _filter_find(smman,*s);
+        if (filter)
+          goto add_rule;
+
+        DBG("No filter found for %s, creating new one", *s);
+        filter = calloc(1, sizeof(Filter));
+        filter->sf = spy_file_new(smman->spy, *s);
+        spy_file_data_set(filter->sf, filter);
+        filter->filename = strdup(*s);
+        filter->rules = eina_hash_string_superfast_new(_filter_rules_free);
+        smman->filters = eina_inlist_append(smman->filters,
+                                            EINA_INLIST_GET(filter));
+
+        add_rule:
+        DBG("Adding rule[%p][%s] to filter[%p][%s]",
+            rule, rule->name, filter, filter->filename);
+        eina_hash_add(filter->rules, rule->name, rule);
+     }
 }
 
 void
@@ -33,7 +91,6 @@ filter_load_done(void *data,
    DBG("smman[%p] rules[%p]", smman, rules);
 }
 
-
 void
 filter_load_error(void *data,
                   Rules *rules,
diff --git a/src/bin/log.c b/src/bin/log.c
new file mode 100644
index 0000000..4ebc053
--- /dev/null
+++ b/src/bin/log.c
@@ -0,0 +1,113 @@
+#include "smman.h"
+
+typedef struct _Log
+{
+   const char *filename,
+              *source_host,
+              *source_path,
+              *message;
+   Eina_Strbuf *tags;
+   Eina_Bool todel;
+} Log;
+
+Eina_Bool
+_log_line_match(const char *log, Rule *rule)
+{
+   Rule_Regex *rr;
+   Eina_Bool excluded = EINA_FALSE;
+   int r;
+
+   EINA_INLIST_FOREACH(rule->spec.regex, rr)
+     {
+        size_t nmatch = 2;
+        regmatch_t pmatch[nmatch];
+
+        r = regexec(&(rr->preg), log, nmatch, pmatch, 0);
+        if (r == rr->must_match)
+          {
+             NFO("Log \"%s\" is not affected by rule %s", log, rule->name);
+             excluded = EINA_TRUE;
+             break;
+          }
+     }
+   return (excluded) ? EINA_FALSE : EINA_TRUE;
+}
+
+Eina_Bool
+log_line_event(void *data,
+               int type EINA_UNUSED,
+               void *event)
+{
+   Smman *smman = data;
+   Spy_Line *sl = event;
+   Spy_File *sf = spy_line_spyfile_get(sl);
+   Filter *filter = spy_file_data_get(sf);
+   Log *log;
+   Rule *rule;
+   Eina_Iterator *it;
+   Eina_Hash_Tuple *t;
+   Eina_Bool r;
+
+   DBG("smman[%p] sl[%p][%s] filter[%p][%s]",
+       smman, sl, spy_file_name_get(sf), filter, filter->filename);
+
+   log = calloc(1, sizeof(Log));
+   eina_stringshare_replace(&log->message, spy_line_get(sl));
+   eina_stringshare_replace(&log->filename, filter->filename);
+   eina_stringshare_replace(&log->source_host, smman->cfg.host);
+   eina_stringshare_replace(&log->source_path, filter->filename);
+   log->tags = eina_strbuf_new();
+
+   /* Now we apply rules */
+   it = eina_hash_iterator_tuple_new(filter->rules);
+   while (eina_iterator_next(it, (void **)&t))
+     {
+        const char *name;
+
+        name = t->key;
+        rule = t->data;
+        DBG("rule[%p][%s]", rule, name);
+
+        r = _log_line_match(spy_line_get(sl), rule);
+        if (!r)
+          continue;
+
+        if (rule->spec.todel)
+          {
+             log->todel = EINA_TRUE;
+             break;
+          }
+
+        if (rule->spec.source_host)
+          eina_stringshare_replace(&log->source_host, rule->spec.source_host);
+
+        if (rule->spec.source_path)
+          eina_stringshare_replace(&log->source_path, rule->spec.source_path);
+
+        if (rule->spec.tags)
+          {
+             if (eina_strbuf_length_get(log->tags))
+               eina_strbuf_append(log->tags, ",");
+             eina_strbuf_append(log->tags, rule->spec.tags);
+          }
+     }
+   eina_iterator_free(it);
+
+   if (log->todel)
+     return EINA_TRUE;
+
+
+   DBG("Log to index :\n"
+       "\tmessage = %s\n"
+       "\tfilename = %s\n"
+       "\tsource_host = %s\n"
+       "\tsource_path = %s\n"
+       "\ttags = %s\n",
+       log->message,
+       log->filename,
+       log->source_host,
+       log->source_path,
+       eina_strbuf_string_get(log->tags));
+
+   return EINA_TRUE;
+}
diff --git a/src/bin/main.c b/src/bin/main.c
index e7f0ece..6f85a98 100644
--- a/src/bin/main.c
+++ b/src/bin/main.c
@@ -48,7 +48,7 @@ init(void)
 {
    Smman *smman;
 
-   smman_log_dom_global = eina_log_domain_register("smman", EINA_COLOR_YELLOW);
+   smman_log_dom_global = eina_log_domain_register("smman", EINA_COLOR_CYAN);
    if (smman_log_dom_global < 0)
      {
         EINA_LOG_ERR("Smman can not create a general log domain");
@@ -69,6 +69,14 @@ init(void)
         return NULL;
      }
 
+   smman->spy = spy_new();
+   if (!smman->spy)
+     {
+        ERR("Failed to create new spy");
+        return NULL;
+     }
+
+   smman->ev.sl = ecore_event_handler_add(SPY_EVENT_LINE,log_line_event, smman);
    return smman;
 }
 
@@ -103,6 +111,7 @@ int main(int argc, char **argv)
 
    conf_init();
    rules_init();
+   spy_init();
 
    smman = init();
    if (!smman)
diff --git a/src/bin/smman.h b/src/bin/smman.h
index a532b86..d75452b 100644
--- a/src/bin/smman.h
+++ b/src/bin/smman.h
@@ -8,20 +8,36 @@
 #include <Eio.h>
 #include <Conf.h>
 #include <Rules.h>
+#include <Spy.h>
 
 int smman_log_dom_global;
 
 typedef struct _Smman
 {
    Rules *rules;
+   Spy *spy;
+   Eina_Inlist *filters;
 
    struct
    {
       const char *server,
                  *host;
    } cfg;
+
+   struct
+   {
+      Ecore_Event_Handler *sl; /* SPY_EVENT_LINE */
+   } ev;
 } Smman;
 
+typedef struct _Filter
+{
+   EINA_INLIST;
+   const char *filename;
+   Spy_File *sf;
+   Eina_Hash *rules;
+} Filter;
+
 #define ERR(...) EINA_LOG_DOM_ERR(smman_log_dom_global, __VA_ARGS__)
 #define DBG(...) EINA_LOG_DOM_DBG(smman_log_dom_global, __VA_ARGS__)
 #define NFO(...) EINA_LOG_DOM_INFO(smman_log_dom_global, __VA_ARGS__)
@@ -34,3 +50,5 @@ void config_error(void *data, Conf *conf, const char *errstr);
 void filter_load(void *data, Rules *rules, Rule *rule);
 void filter_load_done(void *data, Rules *rules);
 void filter_load_error(void *data, Rules *rules, const char *errstr);
+
+Eina_Bool log_line_event(void *data, int type, void *event);
diff --git a/src/include/Rules.h b/src/include/Rules.h
index e20e2a5..86d93ac 100644
--- a/src/include/Rules.h
+++ b/src/include/Rules.h
@@ -46,4 +46,5 @@ int rules_shutdown(void);
 Rules * rules_new(const char *directory);
 Eina_Bool rules_load(Rules *rules, Rules_Progress_Cb progress_cb, Rules_Done_Cb done_cb, Rules_Error_Cb error_cb, void *data);
 
+void rules_rule_free(Rule *rule);
 #endif
diff --git a/src/include/Spy.h b/src/include/Spy.h
index d1b182d..b24c42b 100644
--- a/src/include/Spy.h
+++ b/src/include/Spy.h
@@ -18,9 +18,10 @@ void spy_free(Spy *spy);
 
 Spy_File * spy_file_new(Spy *spy, const char *file);
 Spy_File * spy_file_get(Spy *spy, const char *file);
-const char * spy_line_get(Spy_Line *sl);
 const char * spy_file_name_get(Spy_File *sf);
 void spy_file_data_set(Spy_File *sf, const void *data);
-const void * spy_file_data_get(Spy_File *sf);
+void * spy_file_data_get(Spy_File *sf);
 
+const char * spy_line_get(Spy_Line *sl);
+Spy_File * spy_line_spyfile_get(Spy_Line *sl);
 #endif
diff --git a/src/lib/Makefile.mk b/src/lib/Makefile.mk
index 860a1b4..bb78459 100644
--- a/src/lib/Makefile.mk
+++ b/src/lib/Makefile.mk
@@ -14,8 +14,9 @@ src_lib_libconf_la_CFLAGS = $(LIBS_CFLAGS) $(EXTRA_CPPFLAGS)
 src_lib_libconf_la_LDFLAGS = $(LIBS_LIBS)
 
 src_lib_libspy_la_SOURCES = \
-src/lib/spy/spy_file.c \
 src/lib/spy/spy_main.c \
+src/lib/spy/spy_file.c \
+src/lib/spy/spy_line.c \
 src/lib/spy/spy_private.h \
 src/include/Spy.h
 src_lib_libspy_la_CFLAGS = $(LIBS_CFLAGS) $(EXTRA_CPPFLAGS)
diff --git a/src/lib/rules/rules_private.h b/src/lib/rules/rules_private.h
index f20293e..1d74d95 100644
--- a/src/lib/rules/rules_private.h
+++ b/src/lib/rules/rules_private.h
@@ -27,8 +27,6 @@ typedef struct _Rules_Load
    } cb;
 } Rules_Load;
 
-void rules_rule_free(Rule *rule);
-
 Eina_Bool rules_load_ls_filter(void *data, Eio_File *handler, const Eina_File_Direct_Info *info);
 void rules_load_ls(void *data, Eio_File *handler, const Eina_File_Direct_Info *info);
 void rules_load_ls_done(void *data, Eio_File *handler);
diff --git a/src/lib/spy/spy_file.c b/src/lib/spy/spy_file.c
index 49efbbf..944f326 100644
--- a/src/lib/spy/spy_file.c
+++ b/src/lib/spy/spy_file.c
@@ -87,7 +87,7 @@ _spy_file_cb(void *data,
         return;
      }
 
-   fseeko(sf->read.fd, sf->read.offset, SEEK_SET);
+   fseeko(sf->read.fd, sf->read.offset - 1, SEEK_SET);
    left = sf->read.length;
 
    while (nbr != (size_t)sf->read.length)
@@ -185,15 +185,9 @@ spy_file_poll(void *data)
 }
 
 const char *
-spy_line_get(Spy_Line *sl)
-{
-   return sl->line;
-}
-
-
-const char *
 spy_file_name_get(Spy_File *sf)
 {
+   EINA_SAFETY_ON_NULL_RETURN_VAL(sf, NULL);
    return sf->name;
 }
 
@@ -204,8 +198,9 @@ spy_file_data_set(Spy_File *sf,
    sf->data = (const void *)data;
 }
 
-const void *
+void *
 spy_file_data_get(Spy_File *sf)
 {
-   return sf->data;
+   EINA_SAFETY_ON_NULL_RETURN_VAL(sf, NULL);
+   return (void *)sf->data;
 }
diff --git a/src/lib/spy/spy_line.c b/src/lib/spy/spy_line.c
new file mode 100644
index 0000000..0768027
--- /dev/null
+++ b/src/lib/spy/spy_line.c
@@ -0,0 +1,14 @@
+#include "spy_private.h"
+
+const char *
+spy_line_get(Spy_Line *sl)
+{
+   return sl->line;
+}
+
+Spy_File *
+spy_line_spyfile_get(Spy_Line *sl)
+{
+   return sl->sf;
+}
+
diff --git a/src/lib/spy/spy_main.c b/src/lib/spy/spy_main.c
index 139afed..bb171b3 100644
--- a/src/lib/spy/spy_main.c
+++ b/src/lib/spy/spy_main.c
@@ -90,7 +90,7 @@ spy_file_new(Spy *spy, const char *file)
      }
 
    sf->poll.size = st.st_size;
-   sf->poll.timer = ecore_timer_add(5.0, spy_file_poll, sf);
+   sf->poll.timer = ecore_timer_add(0.3, spy_file_poll, sf);
 
    spy->files = eina_inlist_append(spy->files, EINA_INLIST_GET(sf));
    DBG("spy_file[%p] size[%zd]", sf, st.st_size);