exactness/src/bin/exactness.c

#include "config.h"
#include <Ecore.h>
#include <Ecore_Getopt.h>
#include <Emile.h>

#include "exactness_private.h"

#define SCHEDULER_CMD_SIZE 1024

#define ORIG_SUBDIR "orig"
#define CURRENT_SUBDIR "current"

#define EXACTNESS_PATH_MAX 1024

#define BUF_SIZE 1024

#define CONFIG "ELM_SCALE=1 ELM_FINGER_SIZE=10"

typedef struct
{
   EINA_INLIST;
   char *name;
   const char *command;
} List_Entry;

typedef enum
{
   RUN_SIMULATION,
   RUN_PLAY,
   RUN_INIT
} Run_Mode;

static unsigned short _running_jobs, _max_jobs;
static char *_base_dir;
static char *_dest_dir;
static char *_wrap_command;
static int _verbose = 0;

static Run_Mode _mode;
static List_Entry *_next_test_to_run = NULL;
static unsigned int _tests_executed = 0;

static Eina_List *_errors;
static Eina_List *_compare_errors;

static Eina_Bool _job_consume();

static void
_printf(int verbose, const char *fmt, ...)
{
   va_list ap;
   if (!_verbose || verbose > _verbose) return;

   va_start(ap, fmt);
   vprintf(fmt, ap);
   va_end(ap);
}

static void
_run_command_prepare(const List_Entry *ent, char *buf)
{
   Eina_Strbuf *sbuf = eina_strbuf_new();
   eina_strbuf_append_printf(sbuf, "TSUITE_VERBOSE=%d ", _verbose);
   eina_strbuf_append_printf(sbuf, "TSUITE_FILE_NAME='%s/%s.rec' ",
         _base_dir, ent->name);
   eina_strbuf_append_printf(sbuf, "TSUITE_TEST_NAME='%s' ", ent->name);
   switch (_mode)
     {
      case RUN_PLAY:
           {
              eina_strbuf_append(sbuf, "ELM_ENGINE='buffer' ");
              eina_strbuf_append_printf(sbuf, "TSUITE_DEST_DIR='%s/%s' ",
                    _dest_dir, CURRENT_SUBDIR);
              break;
           }
      case RUN_INIT:
           {
              eina_strbuf_append(sbuf, "ELM_ENGINE='buffer' ");
              eina_strbuf_append_printf(sbuf, "TSUITE_DEST_DIR='%s/%s' ",
                    _dest_dir, ORIG_SUBDIR);
              break;
           }
      default: break;
     }
   eina_strbuf_append_printf(sbuf, "LD_PRELOAD='%s/exactness/libexactness_%s.so' %s %s %s",
         PACKAGE_LIBDIR, "player",
         CONFIG, _wrap_command, ent->command);
   strncpy(buf, eina_strbuf_string_get(sbuf), SCHEDULER_CMD_SIZE-1);
   eina_strbuf_free(sbuf);
   _printf(1, "Command: %s\n", buf);
}

static Eina_Bool
_check_prefix(const char *prefix, const char *name)
{
   unsigned int len = strlen(prefix);
   return (!strncmp(name, prefix, len) && (strlen(name) > len) && (name[len] == SHOT_DELIMITER));
}

static void
_prefix_rm_cb(const char *name, const char *path, void *data)
{
   const char *prefix = data;
   if (_check_prefix(prefix, name))
     {
        char buf[EXACTNESS_PATH_MAX];
        snprintf(buf, EXACTNESS_PATH_MAX, "%s/%s", path, name);
        if (unlink(buf))
          {
             printf("Failed deleting '%s/%s': ", path, name);
             perror("");
          }
     }
}

static void
_run_test_prefix_rm(const char *dir, const char *prefix)
{
   eina_file_dir_list(dir, 0, _prefix_rm_cb, (void *) prefix);
}

static Eina_Bool
_job_deleted_cb(void *data EINA_UNUSED, int type EINA_UNUSED, void *event)
{
   Ecore_Exe_Event_Del *msg = (Ecore_Exe_Event_Del *) event;

   if ((msg->exit_code != 0) || (msg->exit_signal != 0))
    {
        List_Entry *ent = ecore_exe_data_get(msg->exe);
        _errors = eina_list_append(_errors, ent);
     }

   _running_jobs--;

   _job_consume();

   /* If all jobs are done. */
   if (!_running_jobs)
     {
        ecore_main_loop_quit();
        return ECORE_CALLBACK_DONE;
     }

   return ECORE_CALLBACK_RENEW;
}

static Eina_Bool
_job_consume()
{
   static Ecore_Event_Handler *job_del_callback_handler = NULL;
   char buf[SCHEDULER_CMD_SIZE];
   Ecore_Exe *exe;
   List_Entry *ent = _next_test_to_run;

   if (_running_jobs == _max_jobs) return EINA_FALSE;
   if (!ent) return EINA_FALSE;

   _running_jobs++;
   _tests_executed++;

   _run_command_prepare(ent, buf);
   switch (_mode)
     {
      case RUN_PLAY:
           {
              _run_test_prefix_rm(CURRENT_SUBDIR, ent->name);
              _printf(1, "Running %s\n", ent->name);
              break;
           }
      case RUN_INIT:
           {
              _run_test_prefix_rm(ORIG_SUBDIR, ent->name);
              break;
           }
      default: break;
     }

   if (!job_del_callback_handler)
     {
        job_del_callback_handler = ecore_event_handler_add(ECORE_EXE_EVENT_DEL,
              _job_deleted_cb, NULL);
     }

   exe = ecore_exe_pipe_run(buf, ECORE_EXE_TERM_WITH_PARENT, ent);

   _next_test_to_run = EINA_INLIST_CONTAINER_GET(
         EINA_INLIST_GET(ent)->next, List_Entry);

   if (!exe)
     {
        fprintf(stderr, "Failed executing test '%s'\n", ent->name);
     }

   return EINA_TRUE;
}

static void
_scheduler_run()
{
   while (_job_consume());
}

static Eina_Bool
_file_sha1_get(const char *filename, unsigned char *result)
{
   Eina_File *f = NULL;
   const char *key = "0123456789abcde";
   int key_len = strlen(key);
   unsigned int size = 0;
   Eina_Binbuf *buf = NULL;
   void *data = NULL;

   f = eina_file_open(filename, EINA_FALSE);
   if (!f) goto false;
   size = eina_file_size_get(f);
   if (size < 1) goto false;
   data = eina_file_map_all(f, EINA_FILE_POPULATE);
   if (!data) goto false;

   buf = eina_binbuf_manage_new(data, size, EINA_TRUE);
   if (!buf)
     {
        fprintf(stderr, "Could not create Binary Buffer");
        goto false;
     }
   if (!emile_binbuf_hmac_sha1(key, key_len, buf, result))
     {
        fprintf(stderr, "Cannot generate sha1 for image");
        goto false;
     }
   eina_binbuf_free(buf);
   eina_file_close(f);
   return EINA_TRUE;

false:
   if (buf) eina_binbuf_free(buf);
   if (f) eina_file_close(f);
   return EINA_FALSE;
}

#define _DIGEST_SIZE 20
static Eina_Bool
_is_equal(const char *filename1, const char *filename2)
{
   unsigned char res1[_DIGEST_SIZE], res2[_DIGEST_SIZE];
   if (!_file_sha1_get(filename1, res1))
      return EINA_FALSE;
   if (!_file_sha1_get(filename2, res2))
      return EINA_FALSE;

   return !memcmp(res1, res2, _DIGEST_SIZE);
}

static void
_compare_list_cb(const char *name, const char *path EINA_UNUSED, void *data)
{
   const char *prefix = data;
   if (_check_prefix(prefix, name))
     {
        char filename1[EXACTNESS_PATH_MAX], filename2[EXACTNESS_PATH_MAX];
        snprintf(filename1, EXACTNESS_PATH_MAX, "%s/%s/%s", _dest_dir, ORIG_SUBDIR, name);
        snprintf(filename2, EXACTNESS_PATH_MAX, "%s/%s/%s", _dest_dir, CURRENT_SUBDIR, name);
        if (!_is_equal(filename1, filename2))
          {
             char buf[EXACTNESS_PATH_MAX];
             _compare_errors = eina_list_append(_compare_errors, strdup(name));

             /* FIXME: Clean up. */
             snprintf(buf, EXACTNESS_PATH_MAX,
                   "compare '%s' '%s' '%s/%s/comp_%s'",
                   filename1, filename2,
                   _dest_dir,
                   CURRENT_SUBDIR, name);
             if (system(buf))
               {
                  fprintf(stderr, "Failed image comparing '%s'\n", name);
               }
          }
     }
}

static void
_run_test_compare(const List_Entry *ent)
{
   char origdir[EXACTNESS_PATH_MAX];
   snprintf(origdir, EXACTNESS_PATH_MAX, "%s/%s", _dest_dir, ORIG_SUBDIR);
   eina_file_dir_list(origdir, 0, _compare_list_cb, ent->name);
}

static List_Entry *
_list_file_load(const char *filename)
{
   List_Entry *ret = NULL;
   char buf[BUF_SIZE] = "";
   FILE *file;
   file = fopen(filename, "r");
   if (!file)
     {
        perror("Failed opening list file");
        return NULL;
     }

   while (fgets(buf, BUF_SIZE, file))
     {
        /* Skip comment/empty lines. */
        if ((*buf == '#') || (*buf == '\n') || (!*buf))
           continue;

        char *tmp;
        List_Entry *cur = calloc(1, sizeof(*cur));
        cur->name = strdup(buf);

        /* Set the command to the second half and put a \0 in between. */
        tmp = strchr(cur->name, ' ');
        if (tmp)
          {
             *tmp = '\0';
             cur->command = tmp + 1;
          }
        else
          {
             /* FIXME: error. */
             cur->command = "";
          }

        /* Replace the newline char with a \0. */
        tmp = strchr(cur->command, '\n');
        if (tmp)
          {
             *tmp = '\0';
          }

        ret = EINA_INLIST_CONTAINER_GET(
              eina_inlist_append(EINA_INLIST_GET(ret), EINA_INLIST_GET(cur)),
              List_Entry);
     }

   return ret;
}

static void
_list_file_free(List_Entry *list)
{
   while (list)
     {
        List_Entry *ent = list;
        list = EINA_INLIST_CONTAINER_GET(EINA_INLIST_GET(list)->next,
              List_Entry);

        free(ent->name);
        free(ent);
        /* we don't free ent->command because it's allocated together. */
     }
}

static int
_errors_sort_cb(List_Entry *a, List_Entry *b)
{
   return strcmp(a->name, b->name);
}

static const Ecore_Getopt optdesc = {
  "exactness",
  "%prog [options] <-r|-p|-i|-s> <list file>",
  PACKAGE_VERSION,
  "(C) 2013 Enlightenment",
  "BSD",
  "A pixel perfect test suite for EFL based applications.",
  0,
  {
    ECORE_GETOPT_STORE_STR('b', "base-dir", "The location of the rec files."),
    ECORE_GETOPT_STORE_STR('d', "dest-dir", "The location of the images."),
    ECORE_GETOPT_STORE_STR('w', "wrap", "Use a custom command to launch the tests (e.g valgrind)."),
    ECORE_GETOPT_STORE_USHORT('j', "jobs", "The number of jobs to run in parallel."),
    ECORE_GETOPT_STORE_TRUE('p', "play", "Run in play mode."),
    ECORE_GETOPT_STORE_TRUE('i', "init", "Run in init mode."),
    ECORE_GETOPT_STORE_TRUE('s', "simulation", "Run in simulation mode."),
    ECORE_GETOPT_COUNT('v', "verbose", "Turn verbose messages on."),

    ECORE_GETOPT_LICENSE('L', "license"),
    ECORE_GETOPT_COPYRIGHT('C', "copyright"),
    ECORE_GETOPT_VERSION('V', "version"),
    ECORE_GETOPT_HELP('h', "help"),
    ECORE_GETOPT_SENTINEL
  }
};

int
main(int argc, char *argv[])
{
   int ret = 0;
   List_Entry *test_list;
   int args = 0;
   const char *list_file = "";
   char tmp[EXACTNESS_PATH_MAX];
   Eina_Bool mode_play = EINA_FALSE, mode_init = EINA_FALSE, mode_simulation = EINA_FALSE;
   Eina_Bool want_quit = EINA_FALSE;
   Ecore_Getopt_Value values[] = {
     ECORE_GETOPT_VALUE_STR(_base_dir),
     ECORE_GETOPT_VALUE_STR(_dest_dir),
     ECORE_GETOPT_VALUE_STR(_wrap_command),
     ECORE_GETOPT_VALUE_USHORT(_max_jobs),
     ECORE_GETOPT_VALUE_BOOL(mode_play),
     ECORE_GETOPT_VALUE_BOOL(mode_init),
     ECORE_GETOPT_VALUE_BOOL(mode_simulation),
     ECORE_GETOPT_VALUE_INT(_verbose),

     ECORE_GETOPT_VALUE_BOOL(want_quit),
     ECORE_GETOPT_VALUE_BOOL(want_quit),
     ECORE_GETOPT_VALUE_BOOL(want_quit),
     ECORE_GETOPT_VALUE_BOOL(want_quit),
     ECORE_GETOPT_VALUE_NONE
   };

   ecore_init();
   mode_play = mode_init = mode_simulation = EINA_FALSE;
   want_quit = EINA_FALSE;
   _base_dir = "./recordings";
   _dest_dir = "./";
   _wrap_command = "";
   _max_jobs = 1;
   _verbose = 0;

   args = ecore_getopt_parse(&optdesc, values, argc, argv);
   if (args < 0)
     {
        fprintf(stderr, "Failed parsing arguments.\n");
        ret = 1;
        goto end;
     }
   else if (want_quit)
     {
        ret = 1;
        goto end;
     }
   else if (args == argc)
     {
        fprintf(stderr, "Expected test list as the last argument..\n");
        ecore_getopt_help(stderr, &optdesc);
        ret = 1;
        goto end;
     }
   else if (mode_play + mode_init + mode_simulation != 1)
     {
        fprintf(stderr, "At least and only one of the running modes can be set.\n");
        ecore_getopt_help(stderr, &optdesc);
        ret = 1;
        goto end;
     }

   list_file = argv[args];


   /* Load the list file and start iterating over the records. */
   test_list = _list_file_load(list_file);
   _next_test_to_run = test_list;

   if (!test_list)
     {
        fprintf(stderr, "No matching tests found in '%s'\n", list_file);
        ret = 1;
        goto end;
     }

   /* Pre-run summary */
   fprintf(stderr, "Running with settings:\n");
   fprintf(stderr, "\tConcurrent jobs: %d\n", _max_jobs);
   fprintf(stderr, "\tTest list: %s\n", list_file);
   fprintf(stderr, "\tBase dir: %s\n", _base_dir);
   fprintf(stderr, "\tDest dir: %s\n", _dest_dir);

   if (mode_play)
     {
        _mode = RUN_PLAY;
        if (snprintf(tmp, EXACTNESS_PATH_MAX, "%s/%s", _dest_dir, CURRENT_SUBDIR)
            >= EXACTNESS_PATH_MAX)
          {
             fprintf(stderr, "Path too long: %s", tmp);
             ret = 1;
             goto end;
          }
        mkdir(tmp, 0744);
     }
   else if (mode_init)
     {
        _mode = RUN_INIT;
        if (snprintf(tmp, EXACTNESS_PATH_MAX, "%s/%s", _dest_dir, ORIG_SUBDIR)
            >= EXACTNESS_PATH_MAX)
          {
             fprintf(stderr, "Path too long: %s", tmp);
             ret = 1;
             goto end;
          }
        mkdir(tmp, 0744);
     }
   else if (mode_simulation)
     {
        _mode = RUN_SIMULATION;
     }
   _scheduler_run();


   ecore_main_loop_begin();

   /* Results */
   printf("*******************************************************\n");
   if (mode_play)
     {
        List_Entry *list_itr;

        EINA_INLIST_FOREACH(test_list, list_itr)
          {
             _run_test_compare(list_itr);
          }
     }

   printf("Finished executing %u out of %u tests.\n",
         _tests_executed,
         eina_inlist_count(EINA_INLIST_GET(test_list)));

   /* Sort the errors and the compare_errors. */
   _errors = eina_list_sort(_errors, 0, (Eina_Compare_Cb) _errors_sort_cb);
   _compare_errors = eina_list_sort(_compare_errors, 0, (Eina_Compare_Cb) strcmp);

   if (_errors || _compare_errors)
     {
        FILE *report_file;
        char report_filename[EXACTNESS_PATH_MAX] = "";
        /* Generate the filename. */
        snprintf(report_filename, EXACTNESS_PATH_MAX,
              "%s/%s/errors.html",
              _dest_dir, mode_init ? ORIG_SUBDIR : CURRENT_SUBDIR);
        report_file = fopen(report_filename, "w+");
        if (report_file)
          {
             printf("%s %p\n", report_filename, report_file);
             fprintf(report_file,
                   "<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">"
                   "<html xmlns=\"http://www.w3.org/1999/xhtml\"><head><title>Exactness report</title></head><body>");

             if (_errors)
               {
                  fprintf(report_file,
                        "<h1>Tests that failed execution:</h1><ul>");
                  Eina_List *itr;
                  List_Entry *ent;
                  printf("List of tests that failed execution:\n");
                  EINA_LIST_FOREACH(_errors, itr, ent)
                    {
                       printf("\t* %s\n", ent->name);

                       fprintf(report_file, "<li>%s</li>", ent->name);
                    }
                  fprintf(report_file, "</ul>");
               }

             if (_compare_errors)
               {
                  fprintf(report_file,
                        "<h1>Images that failed comparison: (Original, Current, Diff)</h1><ul>");
                  char *test_name;
                  printf("List of images that failed comparison:\n");
                  EINA_LIST_FREE(_compare_errors, test_name)
                    {
                       printf("\t* %s\n", test_name);

                       fprintf(report_file, "<li><h2>%s</h2> <img src='../orig/%s' alt='Original' /> <img src='%s' alt='Current' /> <img src='comp_%s' alt='Diff' /></li>", test_name, test_name, test_name, test_name);
                       free(test_name);
                    }
                  fprintf(report_file, "</ul>");
               }
             fprintf(report_file,
                   "</body></html>");

             printf("Report html: %s\n", report_filename);
             ret = 1;
          }
        else
          {
             perror("Failed opening report file");
          }
     }

   _list_file_free(test_list);
end:
   ecore_shutdown();

   return ret;
}