exactness/src/bin/exactness.c

#include "config.h"
#include <Ecore.h>
#include <Ecore_Getopt.h>
#include <Emile.h>

#include "exactness_private.h"


typedef struct _List_Entry List_Entry;

struct _List_Entry
{
   EINA_INLIST;
   char *name;
   const char *command;
};

typedef struct _Exactness_Config Exactness_Config;

struct _Exactness_Config
{
   unsigned short jobs;
   char *base_dir;
   char *dest_dir;
   char *wrap_command;
   Eina_Bool verbose;
   Eina_Bool store_objects;
};

typedef struct _Exactness_Ctx Exactness_Ctx;

struct _Exactness_Ctx
{
   unsigned int tests_executed;
   Eina_List *errors;
   Eina_List *compare_errors;
};

#define SCHEDULER_CMD_SIZE 1024

typedef void (*Scheduler_Cb)(const List_Entry *, char *);

Exactness_Config exactness_config;
Exactness_Ctx exactness_ctx;

#define ORIG_SUBDIR "orig"
#define CURRENT_SUBDIR "current"

#define EXACTNESS_PATH_MAX 1024

#define BUF_SIZE 1024

#define CONFIG "ELM_SCALE=1 ELM_FINGER_SIZE=10"

typedef enum
{
   RUN_SIMULATION,
   RUN_PLAY,
   RUN_RECORD,
   RUN_INIT
} Run_Mode;

typedef struct
{
   Scheduler_Cb prepare_func;
   List_Entry *last;
   unsigned short jobs;
} Scheduler_Ctx;

static Ecore_Event_Handler *_job_del_callback_handler = NULL;

static Eina_Bool _job_dispatch(List_Entry *ent, Scheduler_Ctx *ctx);

static Eina_Bool
_job_deleted_cb(void *data, int type EINA_UNUSED, void *event)
{
   Ecore_Exe_Event_Del *msg = (Ecore_Exe_Event_Del *) event;
   Scheduler_Ctx *ctx = data;

   if ((msg->exit_code != 0) || (msg->exit_signal != 0))
     {
        List_Entry *ent = ecore_exe_data_get(msg->exe);
        exactness_ctx.errors = eina_list_append(exactness_ctx.errors, ent);
     }

   ctx->jobs++;

   exactness_ctx.tests_executed++;

   if (ctx->last && EINA_INLIST_GET(ctx->last)->next)
     {
        ctx->last = EINA_INLIST_CONTAINER_GET(
              EINA_INLIST_GET(ctx->last)->next, List_Entry);

        _job_dispatch(ctx->last, ctx);
     }

   /* If all jobs are done. */
   if (ctx->jobs == exactness_config.jobs)
     {
        free(ctx);
        ecore_main_loop_quit();
        return ECORE_CALLBACK_DONE;
     }

   return ECORE_CALLBACK_RENEW;
}

static Eina_Bool
_job_dispatch(List_Entry *ent, Scheduler_Ctx *ctx)
{
   char buf[SCHEDULER_CMD_SIZE];
   Ecore_Exe *exe;

   if (ctx->jobs == 0)
      return EINA_FALSE;
   ctx->jobs--;

   ctx->prepare_func(ent, buf);

   if (!_job_del_callback_handler)
     {
        _job_del_callback_handler = ecore_event_handler_add(ECORE_EXE_EVENT_DEL,
              _job_deleted_cb, ctx);
     }

   exe = ecore_exe_pipe_run(buf, ECORE_EXE_TERM_WITH_PARENT, ent);

   if (!exe)
     {
        fprintf(stderr, "Failed executing test '%s'\n", ent->name);
     }

   return EINA_TRUE;
}

void
scheduler_run(Scheduler_Cb prepare_func, List_Entry *list)
{
   Scheduler_Ctx *ctx = calloc(1, sizeof(*ctx));
   List_Entry *list_itr;
   ctx->jobs = exactness_config.jobs;
   ctx->prepare_func = prepare_func;

   EINA_INLIST_FOREACH(list, list_itr)
     {
        if (!_job_dispatch(list_itr, ctx))
           break;
        ctx->last = list_itr;
     }
}
static void
_run_command_prepare(const List_Entry *ent, Run_Mode mode, char *buf)
{
   Eina_Strbuf *sbuf = eina_strbuf_new();
   eina_strbuf_append_printf(sbuf, "TSUITE_VERBOSE=%d ", exactness_config.verbose);
   eina_strbuf_append_printf(sbuf, "TSUITE_FILE_NAME='%s/%s.rec' ",
         exactness_config.base_dir, ent->name);
   eina_strbuf_append_printf(sbuf, "TSUITE_TEST_NAME='%s' ", ent->name);
   switch (mode)
     {
      case RUN_SIMULATION:
           {
              break;
           }
      case RUN_PLAY:
           {
              eina_strbuf_append(sbuf, "ELM_ENGINE='buffer' ");
              eina_strbuf_append_printf(sbuf, "TSUITE_DEST_DIR='%s/%s' ",
                    exactness_config.dest_dir, CURRENT_SUBDIR);
              if (exactness_config.store_objects)
                 eina_strbuf_append(sbuf, "TSUITE_STORE_OBJECTS=1 ");
              break;
           }
      case RUN_INIT:
           {
              eina_strbuf_append(sbuf, "ELM_ENGINE='buffer' ");
              eina_strbuf_append_printf(sbuf, "TSUITE_DEST_DIR='%s/%s' ",
                    exactness_config.dest_dir, ORIG_SUBDIR);
              if (exactness_config.store_objects)
                 eina_strbuf_append(sbuf, "TSUITE_STORE_OBJECTS=1 ");
              break;
           }
      case RUN_RECORD:
           {
              eina_strbuf_append_printf(sbuf, "TSUITE_DEST_DIR='%s' ",
                    exactness_config.dest_dir);
              break;
           }
      default: break;
     }
   eina_strbuf_append_printf(sbuf, "LD_PRELOAD='%s/exactness/libexactness_%s.so' %s %s %s",
         PACKAGE_LIBDIR, mode == RUN_RECORD ? "recorder" : "player",
         CONFIG, exactness_config.wrap_command, ent->command);
   strncpy(buf, eina_strbuf_string_get(sbuf), SCHEDULER_CMD_SIZE-1);
   eina_strbuf_free(sbuf);
   if (exactness_config.verbose) printf("Command: %s\n", buf);
}

void
run_test_simulation(const List_Entry *ent, char *buf)
{
   _run_command_prepare(ent, RUN_SIMULATION, buf);
}

static Eina_Bool
_check_prefix(const char *prefix, const char *name)
{
   unsigned int len = strlen(prefix);
   return (!strncmp(name, prefix, len) && (strlen(name) > len) && (name[len] == SHOT_DELIMITER));
}

static void
_prefix_rm_cb(const char *name, const char *path, void *data)
{
   const char *prefix = data;
   if (_check_prefix(prefix, name))
     {
        char buf[EXACTNESS_PATH_MAX];
        snprintf(buf, EXACTNESS_PATH_MAX, "%s/%s", path, name);
        if (unlink(buf))
          {
             printf("Failed deleting '%s/%s': ", path, name);
             perror("");
          }
     }
}

void
run_test_prefix_rm(const char *dir, const char *prefix)
{
   eina_file_dir_list(dir, 0, _prefix_rm_cb, (void *) prefix);
}

void
run_test_play(const List_Entry *ent, char *buf)
{
   _run_command_prepare(ent, RUN_PLAY, buf);

   run_test_prefix_rm(CURRENT_SUBDIR, ent->name);
   if (exactness_config.verbose)
     {
        printf("Running %s\n", ent->name);
     }
}

void
run_test_record(const List_Entry *ent, char *buf)
{
   _run_command_prepare(ent, RUN_RECORD, buf);
}

void
run_test_init(const List_Entry *ent, char *buf)
{
   _run_command_prepare(ent, RUN_INIT, buf);

   run_test_prefix_rm(ORIG_SUBDIR, ent->name);
}

static Eina_Bool
_file_sha1_get(const char *filename, unsigned char *result)
{
   Eina_File *f = NULL;
   const char *key = "0123456789abcde";
   int key_len = strlen(key);
   unsigned int size = 0;
   Eina_Binbuf *buf = NULL;
   void *data = NULL;

   f = eina_file_open(filename, EINA_FALSE);
   if (!f) goto false;
   size = eina_file_size_get(f);
   if (size < 1) goto false;
   data = eina_file_map_all(f, EINA_FILE_POPULATE);
   if (!data) goto false;

   buf = eina_binbuf_manage_new(data, size, EINA_TRUE);
   if (!buf)
     {
        fprintf(stderr, "Could not create Binary Buffer");
        goto false;
     }
   if (!emile_binbuf_hmac_sha1(key, key_len, buf, result))
     {
        fprintf(stderr, "Cannot generate sha1 for image");
        goto false;
     }
   eina_binbuf_free(buf);
   eina_file_close(f);
   return EINA_TRUE;

false:
   if (buf) eina_binbuf_free(buf);
   if (f) eina_file_close(f);
   return EINA_FALSE;
}

#define _DIGEST_SIZE 20
static Eina_Bool
_is_equal(const char *filename1, const char *filename2)
{
   unsigned char res1[_DIGEST_SIZE], res2[_DIGEST_SIZE];
   if (!_file_sha1_get(filename1, res1))
      return EINA_FALSE;
   if (!_file_sha1_get(filename2, res2))
      return EINA_FALSE;

   return !memcmp(res1, res2, _DIGEST_SIZE);
}

static void
_compare_list_cb(const char *name, const char *path EINA_UNUSED, void *data)
{
   const char *prefix = data;
   if (_check_prefix(prefix, name))
     {
        char filename1[EXACTNESS_PATH_MAX], filename2[EXACTNESS_PATH_MAX];
        snprintf(filename1, EXACTNESS_PATH_MAX, "%s/%s/%s", exactness_config.dest_dir, ORIG_SUBDIR, name);
        snprintf(filename2, EXACTNESS_PATH_MAX, "%s/%s/%s", exactness_config.dest_dir, CURRENT_SUBDIR, name);
        if (!_is_equal(filename1, filename2))
          {
             char buf[EXACTNESS_PATH_MAX];
             exactness_ctx.compare_errors =
                eina_list_append(exactness_ctx.compare_errors,
                      strdup(name));

             /* FIXME: Clean up. */
             snprintf(buf, EXACTNESS_PATH_MAX,
                   "compare '%s' '%s' '%s/%s/comp_%s'",
                   filename1, filename2,
                   exactness_config.dest_dir,
                   CURRENT_SUBDIR, name);
             if (system(buf))
               {
                  fprintf(stderr, "Failed image comparing '%s'\n", name);
               }
          }
     }
}

void
run_test_compare(const List_Entry *ent)
{
   char origdir[EXACTNESS_PATH_MAX];
   snprintf(origdir, EXACTNESS_PATH_MAX, "%s/%s", exactness_config.dest_dir, ORIG_SUBDIR);
   eina_file_dir_list(origdir, 0, _compare_list_cb, ent->name);
}

List_Entry *
list_file_load(const char *filename)
{
   List_Entry *ret = NULL;
   char buf[BUF_SIZE] = "";
   FILE *file;
   file = fopen(filename, "r");
   if (!file)
     {
        perror("Failed opening list file");
        return NULL;
     }

   while (fgets(buf, BUF_SIZE, file))
     {
        /* Skip comment/empty lines. */
        if ((*buf == '#') || (*buf == '\n') || (!*buf))
           continue;

        char *tmp;
        List_Entry *cur = calloc(1, sizeof(*cur));
        cur->name = strdup(buf);

        /* Set the command to the second half and put a \0 in between. */
        tmp = strchr(cur->name, ' ');
        if (tmp)
          {
             *tmp = '\0';
             cur->command = tmp + 1;
          }
        else
          {
             /* FIXME: error. */
             cur->command = "";
          }

        /* Replace the newline char with a \0. */
        tmp = strchr(cur->command, '\n');
        if (tmp)
          {
             *tmp = '\0';
          }

        ret = EINA_INLIST_CONTAINER_GET(
              eina_inlist_append(EINA_INLIST_GET(ret), EINA_INLIST_GET(cur)),
              List_Entry);
     }

   return ret;
}

void
list_file_free(List_Entry *list)
{
   while (list)
     {
        List_Entry *ent = list;
        list = EINA_INLIST_CONTAINER_GET(EINA_INLIST_GET(list)->next,
              List_Entry);

        free(ent->name);
        free(ent);
        /* we don't free ent->command because it's allocated together. */
     }
}

static int
_errors_sort_cb(List_Entry *a, List_Entry *b)
{
   return strcmp(a->name, b->name);
}

static void
_exactness_errors_sort(Exactness_Ctx *ctx)
{
   ctx->errors =
      eina_list_sort(ctx->errors, 0, (Eina_Compare_Cb) _errors_sort_cb);
   ctx->compare_errors =
      eina_list_sort(ctx->compare_errors, 0, (Eina_Compare_Cb) strcmp);
}

static const Ecore_Getopt optdesc = {
  "exactness",
  "%prog [options] <-r|-p|-i|-s> <list file>",
  PACKAGE_VERSION,
  "(C) 2013 Enlightenment",
  "BSD",
  "A pixel perfect test suite for EFL based applications.",
  0,
  {
    ECORE_GETOPT_STORE_STR('b', "base-dir", "The location of the rec files."),
    ECORE_GETOPT_STORE_STR('d', "dest-dir", "The location of the images."),
    ECORE_GETOPT_STORE_STR('w', "wrap", "Use a custom command to launch the tests (e.g valgrind)."),
    ECORE_GETOPT_STORE_USHORT('j', "jobs", "The number of jobs to run in parallel."),
    ECORE_GETOPT_STORE_TRUE('r', "record", "Run in record mode."),
    ECORE_GETOPT_STORE_TRUE('p', "play", "Run in play mode."),
    ECORE_GETOPT_STORE_TRUE('i', "init", "Run in init mode."),
    ECORE_GETOPT_STORE_TRUE('s', "simulation", "Run in simulation mode."),
    ECORE_GETOPT_STORE_TRUE('S', "store-objects", "Store information about objects at every screen shot time."),
    ECORE_GETOPT_STORE_TRUE('v', "verbose", "Turn verbose messages on."),

    ECORE_GETOPT_LICENSE('L', "license"),
    ECORE_GETOPT_COPYRIGHT('C', "copyright"),
    ECORE_GETOPT_VERSION('V', "version"),
    ECORE_GETOPT_HELP('h', "help"),
    ECORE_GETOPT_SENTINEL
  }
};

int
main(int argc, char *argv[])
{
   int ret = 0;
   List_Entry *test_list;
   int args = 0;
   const char *list_file = "";
   char tmp[EXACTNESS_PATH_MAX];
   Eina_Bool mode_record, mode_play, mode_init, mode_simulation;
   Eina_Bool want_quit;
   Ecore_Getopt_Value values[] = {
     ECORE_GETOPT_VALUE_STR(exactness_config.base_dir),
     ECORE_GETOPT_VALUE_STR(exactness_config.dest_dir),
     ECORE_GETOPT_VALUE_STR(exactness_config.wrap_command),
     ECORE_GETOPT_VALUE_USHORT(exactness_config.jobs),
     ECORE_GETOPT_VALUE_BOOL(mode_record),
     ECORE_GETOPT_VALUE_BOOL(mode_play),
     ECORE_GETOPT_VALUE_BOOL(mode_init),
     ECORE_GETOPT_VALUE_BOOL(mode_simulation),
     ECORE_GETOPT_VALUE_BOOL(exactness_config.store_objects),
     ECORE_GETOPT_VALUE_BOOL(exactness_config.verbose),

     ECORE_GETOPT_VALUE_BOOL(want_quit),
     ECORE_GETOPT_VALUE_BOOL(want_quit),
     ECORE_GETOPT_VALUE_BOOL(want_quit),
     ECORE_GETOPT_VALUE_BOOL(want_quit),
     ECORE_GETOPT_VALUE_NONE
   };

   ecore_init();
   mode_record = mode_play = mode_init = mode_simulation = EINA_FALSE;
   want_quit = EINA_FALSE;
   exactness_config.base_dir = "./recordings";
   exactness_config.dest_dir = "./";
   exactness_config.wrap_command = "";
   exactness_config.jobs = 1;
   exactness_config.verbose = EINA_FALSE;

   args = ecore_getopt_parse(&optdesc, values, argc, argv);
   if (args < 0)
     {
        fprintf(stderr, "Failed parsing arguments.\n");
        ret = 1;
        goto end;
     }
   else if (want_quit)
     {
        ret = 1;
        goto end;
     }
   else if (args == argc)
     {
        fprintf(stderr, "Expected test list as the last argument..\n");
        ecore_getopt_help(stderr, &optdesc);
        ret = 1;
        goto end;
     }
   else if (mode_record + mode_play + mode_init + mode_simulation != 1)
     {
        fprintf(stderr, "At least and only one of the running modes can be set.\n");
        ecore_getopt_help(stderr, &optdesc);
        ret = 1;
        goto end;
     }

   list_file = argv[args];


   /* Load the list file and start iterating over the records. */
   test_list = list_file_load(list_file);

   if (!test_list)
     {
        fprintf(stderr, "No matching tests found in '%s'\n", list_file);
        ret = 1;
        goto end;
     }

   /* Pre-run summary */
   fprintf(stderr, "Running with settings:\n");
   fprintf(stderr, "\tConcurrent jobs: %d\n", exactness_config.jobs);
   fprintf(stderr, "\tTest list: %s\n", list_file);
   fprintf(stderr, "\tBase dir: %s\n", exactness_config.base_dir);
   fprintf(stderr, "\tDest dir: %s\n", exactness_config.dest_dir);

   if (mode_record)
     {
        scheduler_run(run_test_record, test_list);
     }
   else if (mode_play)
     {
        if (snprintf(tmp, EXACTNESS_PATH_MAX, "%s/%s", exactness_config.dest_dir, CURRENT_SUBDIR)
            >= EXACTNESS_PATH_MAX)
          {
             fprintf(stderr, "Path too long: %s", tmp);
             ret = 1;
             goto end;
          }
        mkdir(tmp, 0744);
        scheduler_run(run_test_play, test_list);
     }
   else if (mode_init)
     {
        if (snprintf(tmp, EXACTNESS_PATH_MAX, "%s/%s", exactness_config.dest_dir, ORIG_SUBDIR)
            >= EXACTNESS_PATH_MAX)
          {
             fprintf(stderr, "Path too long: %s", tmp);
             ret = 1;
             goto end;
          }
        mkdir(tmp, 0744);
        scheduler_run(run_test_init, test_list);
     }
   else if (mode_simulation)
     {
        scheduler_run(run_test_simulation, test_list);
     }


   ecore_main_loop_begin();

   /* Results */
   printf("*******************************************************\n");
   if (mode_play)
     {
        List_Entry *list_itr;

        EINA_INLIST_FOREACH(test_list, list_itr)
          {
             run_test_compare(list_itr);
          }
     }

   printf("Finished executing %u out of %u tests.\n",
         exactness_ctx.tests_executed,
         eina_inlist_count(EINA_INLIST_GET(test_list)));

   /* Sort the errors and the compare_errors. */
   _exactness_errors_sort(&exactness_ctx);

   if (exactness_ctx.errors || exactness_ctx.compare_errors)
     {
        FILE *report_file;
        char report_filename[EXACTNESS_PATH_MAX] = "";
        /* Generate the filename. */
        snprintf(report_filename, EXACTNESS_PATH_MAX,
              "%s/%s/errors.html",
              exactness_config.dest_dir, mode_init ? ORIG_SUBDIR : CURRENT_SUBDIR);
        report_file = fopen(report_filename, "w+");
        if (report_file)
          {
             printf("%s %p\n", report_filename, report_file);
             fprintf(report_file,
                   "<?xml version=\"1.0\" encoding=\"UTF-8\"?><!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">"
                   "<html xmlns=\"http://www.w3.org/1999/xhtml\"><head><title>Exactness report</title></head><body>");

             if (exactness_ctx.errors)
               {
                  fprintf(report_file,
                        "<h1>Tests that failed execution:</h1><ul>");
                  Eina_List *itr;
                  List_Entry *ent;
                  printf("List of tests that failed execution:\n");
                  EINA_LIST_FOREACH(exactness_ctx.errors, itr, ent)
                    {
                       printf("\t* %s\n", ent->name);

                       fprintf(report_file, "<li>%s</li>", ent->name);
                    }
                  fprintf(report_file, "</ul>");
               }

             if (exactness_ctx.compare_errors)
               {
                  fprintf(report_file,
                        "<h1>Images that failed comparison: (Original, Current, Diff)</h1><ul>");
                  char *test_name;
                  printf("List of images that failed comparison:\n");
                  EINA_LIST_FREE(exactness_ctx.compare_errors, test_name)
                    {
                       printf("\t* %s\n", test_name);

                       fprintf(report_file, "<li><h2>%s</h2> <img src='../orig/%s' alt='Original' /> <img src='%s' alt='Current' /> <img src='comp_%s' alt='Diff' /></li>", test_name, test_name, test_name, test_name);
                       free(test_name);
                    }
                  fprintf(report_file, "</ul>");
               }
             fprintf(report_file,
                   "</body></html>");

             printf("Report html: %s\n", report_filename);
             ret = 1;
          }
        else
          {
             perror("Failed opening report file");
          }
     }

   list_file_free(test_list);
end:
   ecore_shutdown();

   return ret;
}