summaryrefslogblamecommitdiff
path: root/doc/smman.dox.in
blob: 5273ab04bbe5460e5c2f892440f8d96a8b49454c (plain) (tree)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
















                                                                         
                               










                                                                                


                                         
                                           
                   
           




                                                         


                                                                                


                                                                           












                                                                                       










                                                                              














                                                                  


                                            
                            
                                             

                                           


                                                                     







                                     

                        






                                                                  
                                                 

                             
                             
    
   






                                      

                                  

                  








                                                                          
                              
   



                      
















                                                                           












                                                                      


















                                                                      

                                           
                           
                                            





















                                                                    
                                  





























                                                                      









                                                                         



















                                                                     



                                              










































                                                                         
































                                                                      
/**
 * @mainpage Syslog Message MANager
 *
 * @author Guillaume Friloux <kuri@efl.so>
 * @version @PACKAGE_VERSION@
 *
 * @section INTRODUCTION Introduction
 * SMMan is a gateway between syslog files and an
 * <a href=http://www.elasticsearch.com>ElasticSearch</a> database.<br />
 * SMMan has a few more interesting stuff : He can tag every log he 
 * sees by using defined rules on them.<br />
 * SMMan uses a configuration file, and needs rules files that must be 
 * written by the user of SMMan.<br />
 * SMMan will then use these rules to monitor all the specified logfiles 
 * (using inotify), and extract every new entry.
 * to filter it using the rules and then indexing it in the configured 
 * <a href=http://www.elasticsearch.com>ElasticSearch</a> database.
 * <img src=data/img/intro.png>
 *
 * <br />
 * @section CONFIGURATION Configuration
 * The configuration file has to be in <b>/etc/smman/smman.conf</b><br />
 * For now, there is only 3 configurable variables :
 * @li @b server : URL to
 * <a href=http://www.elasticsearch.com>ElasticSearch</a> database. 
 * SMMan speaks to <a href=http://www.elasticsearch.com>ElasticSearch</a> using 
 * JSON.
 * @li @b host : Allows you to set a different host that the one returned
 *     by command hostname (optionnal).
 *
 * Exemple of configuration file : <br />
 * @code
 * server = http://localhost:9200/logstash/
 * host = BlackStar
 * @endcode
 *
 * <br />
 * @section RULES Writing rules
 * Writing rules is quite easy. SMMan search for rules in
 * <b>/etc/smman/rules.d/</b><br />
 * Check the
 * <a href="https://git.enlightenment.org/devs/kuri/smman.git/tree/rules">rules 
 * directory</a> in the source code to see examples of rules.<br />
 * Basically, rules allows you to write matches about filenames or messages
 * (using globbing/regexp), and set informations like :
 * @li source_host : Set a custom hostname
 * @li tags : Add tags to the message
 * @li delete : Do not index the log, just drop it
 *
 * <br />
 * @section LOGSTASH Why not using logstash ?
 * @li Its written in ruby and i know nothing to ruby (so i cant modify
 *     anything).
 * @li I have been able to make it crash just by deleting a monitored file,
 *     or by sending chars like éàè.
 * @li I seemed (back in 2010) to be too stupid to understand how to automatically tag 
 * messages properlly with grok.
 *
 */


/**
 * @defgroup Lib-Conf Conf Library
 *
 * @section Lib-Conf-Introduction Introduction
 * Conf is an asynchronous library to help loading/parsing basic configuration
 * files.
 * <br />
 * Each configuration line will be stored in an hash table.
 *
 * @section Lib-Conf-Algorithm Algorithm
 * Here is a simplified sequence chart to understand how it works.
 * @msc
 * MainLoop,Thread1;
 * MainLoop=>MainLoop [ label = "conf_load()" ];
 * MainLoop=>MainLoop [ label = "eio_file_map_all()" ];
 * ---                [ label = "Begin loop" ];
 * MainLoop=>Thread1  [ label = "conf_load_map_filter()" ];
 * Thread1=>Thread1   [ label = "conf_load_line_parse()" ];
 * ---                [ label = "End loop" ];
 * Thread1=>MainLoop  [ label = "conf_callback()" ];
 * MainLoop=>MainLoop [ label = "app_callback()" ];
 * @endmsc
 * As you can see, loading of the file, and parsing of it is done
 * in a separate thread to not block the main loop.
 *
 * @section Lib-Conf-Code Code documentation
 * @li @ref Lib-Conf-Functions
 * @li @ref Lib-Conf-Example
 * @defgroup Lib-Conf-Functions API functions
 *
 * @defgroup Lib-Conf-Example Basic Example
 * @section Lib-Conf-Example-Sample-App Sample program
 * Here is an example program using libconf to load and parse a given
 * configuration file :
 * @code
 * #include <Conf.h>
 *
 * void
 * _conf_done(void *data EINA_UNUSED,
 *            Conf *conf)
 * {
 *    Eina_Iterator *it;
 *    printf("done!\n");
 * 
 *    printf("Results :\n");
 *    it = eina_hash_iterator_tuple_new(conf_variables_get(conf));
 *    while (eina_iterator_next(it, &data))
 *      {
 *         Eina_Hash_Tuple *t = data;
 *         const char *variable = t->key;
 *         const char *value = t->data;
 *         printf("\t%s: %s\n", variable, value);
 *      }
 *    eina_iterator_free(it);
 *    ecore_main_loop_quit();
 * }
 * 
 * void
 * _conf_error(void *data EINA_UNUSED,
 *             Conf *conf EINA_UNUSED,
 *             const char *errstr)
 * {
 *    printf("error : %s\n", errstr);
 * }
 * 
 * int main(int argc, char **argv)
 * {
 *    conf_init();
 * 
 *    if (argc != 2)
 *      {
 *         fprintf(stderr, "You need to provide the conf file to load\n");
 *         return 1;
 *      }
 * 
 *    conf_load(argv[1], _conf_done, _conf_error, NULL);
 * 
 *    ecore_main_loop_begin();
 * 
 *    conf_shutdown();
 *    return 0;
 * }
 * @endcode
 * <br /><br />
 * @section Lib-Conf-Example-Sample-Conf Sample configuration file
 * You can try the configuration file below :
 * @code
 * # This rule will match any log message from /var/log/auth.log that
 * # is about a connection using a valid public key for user root, and that
 * # is not from normally authorized IPs.
 * # These logs will be tagged with 'alert' word
 * filename = /var/log/auth.log
 * message    = .*Accepted publickey for root.*
 * message_unmatch =      .*192\.168\.2\.84.*
 * message_unmatch = .*192\.168\.2\.82.*
 * message_unmatch = .*192\.168\.2\.94.*
 *
 *
 * tags = alert
 * @endcode
 */

/**
 * @defgroup Lib-Spy Spy Library
 *
 * @section Lib-Spy-Introduction Introduction
 * Spy is an asynchronous library to ease the detection of new lines
 * inside files.
 * <br />
 * For every new line detected on a spied file, it will generate a
 * @b SPY_EVENT_LINE event to ecore that your application will need to
 * listen for.
 *
 * @section Lib-Spy-Algorithm Algorithm
 * Here is a simplified sequence chart to understand how it works.
 * @msc
 * MainLoop,Thread1;
 * MainLoop=>MainLoop [ label = "spy_file_new()" ];
 * ---                [ label = "Loop until file changes" ];
 * MainLoop=>MainLoop [ label = "spy_file_poll()" ];
 * ---                [ label = "File changed" ];
 * MainLoop=>Thread1  [ label = "_spy_file_cb()" ];
 * Thread1=>Thread1   [ label = "_spy_file_line_extract()" ];
 * Thread1=>MainLoop  [ label = "_spy_file_event()" ];
 * MainLoop=>MainLoop [ label = "ecore_event_add()" ];
 * ---                [ label = "Return to loop on spy_file_poll()" ];
 * @endmsc
 * For each call to spy_file_new(), an ecore timer is created to
 * periodically call spy_file_poll(). <br />
 * Once a changed is observed on the file, a thread is created to get
 * lines added to the file, and create the ecore events.
 *
 * @section Lib-Spy-Code Code documentation
 * @li @ref Lib-Spy-Functions
 * @li @ref Lib-Spy-Example
 * @defgroup Lib-Spy-Functions API functions
 *
 * @defgroup Lib-Spy-Example Basic Example
 * @section Lib-Spy-Example-Sample-App Sample program
 * Here is a sample program that uses lib spy to get new line events
 * about the spied file :
 * @code
 * #include <Spy.h>
 * #include <stdio.h>
 *
 * Eina_Bool
 * _line_event(void *data,
 *            int type EINA_UNUSED,
 *            void *event)
 * {
 *    Spy *spy = data;
 *    Spy_Line *sl = event;
 *
 *    printf("spy[%p] sl[%p]\n", spy, sl);
 *    printf("line = %s\n", (char *)spy_line_get(sl));
 *    return EINA_TRUE;
 * }
 *
 * int main(int argc, char **argv)
 * {
 *    Spy *spy;
 *    Ecore_Event_Handler *eeh;
 *
 *    if (argc != 2)
 *      {
 *         fprintf(stderr, "You need to provide a file to spy\n");
 *         return 1;
 *      }
 *
 *    spy_init();
 *
 *    spy = spy_new();
 *    spy_file_new(spy, argv[1]);
 *
 *    eeh = ecore_event_handler_add(SPY_EVENT_LINE, _line_event, spy);
 *
 *    ecore_main_loop_begin();
 *    ecore_event_handler_del(eeh);
 *
 *    spy_free(spy);
 *    spy_shutdown();
 *
 *    return 0;
 * }
 * @endcode
 * Ask it to spy a file on which you will constantly add new lines
 * to see the events in the @a _line_event() function.<br />
 * Libspy is also able to detect a truncate, to reset its internals
 * and go back to begin of file.
 */

/**
 * @defgroup Lib-Rules Rules Library
 * 
 * @section Lib-Rules-Introduction Introduction
 * Rules is an asynchronous library for the loading of smman rules.<br />
 * These rules are basic configuration files, that we read using the
 * @ref Lib-Conf.<br />
 *
 * @section Lib-Rules-Algorithm Algorithm
 * Here is a simplified sequence chart to understand how it works.
 * @msc
 * MainLoop,Thread1,Thread2;
 * MainLoop=>MainLoop [ label = "rules_load()" ];
 * MainLoop=>Thread1  [ label = "eio_file_direct_ls()" ];
 * Thread1=>MainLoop  [ label = "rules_load_ls()" ];
 * ---                [ label = "Loop for every rule found" ];
 * MainLoop=>Thread2  [ label = "conf_load()" ];
 * Thread2=>MainLoop  [ label = "rules_load_rule()" ];
 * MainLoop=>MainLoop [ label = "app_callback()" ];
 * ---                [ label = "End of loop" ];
 * Thread1=>MainLoop  [ label = "rules_load_ls_done()" ];
 * MainLoop=>MainLoop [ label = "app_callback()" ];
 * @endmsc
 * For each call to spy_file_new(), an ecore timer is created to
 * periodically call spy_file_poll(). <br />
 * Once a changed is observed on the file, a thread is created to get
 * lines added to the file, and create the ecore events.
 *
 * @section Lib-Rules-Code Code documentation
 * @li @ref Lib-Rules-Functions
 * @defgroup Lib-Rules-Functions API functions
 */


/**
 * @defgroup Lib-Store Store Library
 *
 * @section Lib-Store-Introduction Introduction
 * Store is an asynchronous library for the sending of data to an HTTP
 * URL.
 *
 * @section Lib-Store-Algorithm
 * All calls in this library are made inside the main loop.<br />
 * We do so because we dont have any heavy operations to do like I/O,
 * or heavy work on strings. We just send data and wait for server
 * to answer.
 * @msc
 * App,Store,Ecore,Ecore_Con;
 * App=>Store       [ label = "store_add()", URL = "@ref store_add" ];
 * Store=>Ecore     [ label = "ecore_event_handler_add()" ];
 * Store=>Ecore     [ label = "ecore_event_handler_add()" ];
 * Store=>Ecore_Con [ label = "ecore_con_url_post()" ];
 * ---              [ label = "From here, we wait for events" ];
 * Ecore_Con=>Store [ label = "store_event_data()" ];
 * Ecore_Con=>Store [ label = "store_event_complete()" ];
 * Store=>App       [ label = "done_cb()" ];
 * @endmsc
 *
 * @section Lib-Store-Code Code documentation
 * @li @ref Lib-Store-Functions
 * @li @ref Lib-Store-Example
 * @defgroup Lib-Store-Functions API functions
 * @defgroup Lib-Store-Example Basic Example
 * @section Lib-Store-Example-Sample-App Sample program
 * This program will send a given string to a local elasticsearch server.
 * @code
 * #include <Store.h>
 *
 * void
 * done_cb(void *data EINA_UNUSED,
 *         Store *store EINA_UNUSED,
 *         char *answer,
 *         size_t len EINA_UNUSED)
 * {
 *    printf("Done_Cb : %s\n", answer);
 * }
 *
 * void
 * error_cb(void *data EINA_UNUSED,
 *          Store *store EINA_UNUSED,
 *          char *strerr)
 * {
 *    printf("Error_Cb : %s\n", strerr);
 * }
 *
 * int main(int argc EINA_UNUSED, char **argv EINA_UNUSED)
 * {
 *    Store *store;
 *
 *    if (argc != 2)
 *      {
 *         fprintf(stderr, "You need to provide a string to store\n");
 *         return 1;
 *      }
 *
 *    store_init();
 *
 *    store = store_new("http://localhost:9200/logsmanager/");
 *    store_add(store, argv[1], strlen(argv[1]),
 *              done_cb, error_cb, store);
 *
 *    ecore_main_loop_begin();
 *
 *    store_shutdown();
 *    return 0;
 * }
 * @endcode
 */