#include "config.h" #include #include #include #include "exactness_private.h" #define SCHEDULER_CMD_SIZE 1024 #define ORIG_SUBDIR "orig" #define CURRENT_SUBDIR "current" #define EXACTNESS_PATH_MAX 1024 #define BUF_SIZE 1024 #define CONFIG "ELM_SCALE=1 ELM_FINGER_SIZE=10" typedef struct { EINA_INLIST; char *name; const char *command; } List_Entry; typedef enum { RUN_SIMULATION, RUN_PLAY, RUN_INIT } Run_Mode; static unsigned short _running_jobs, _max_jobs; static char *_base_dir; static char *_dest_dir; static char *_wrap_command; static int _verbose = 0; static Run_Mode _mode; static List_Entry *_next_test_to_run = NULL; static unsigned int _tests_executed = 0; static Eina_List *_errors; static Eina_List *_compare_errors; static Eina_Bool _job_consume(); static void _printf(int verbose, const char *fmt, ...) { va_list ap; if (!_verbose || verbose > _verbose) return; va_start(ap, fmt); vprintf(fmt, ap); va_end(ap); } static void _run_command_prepare(const List_Entry *ent, char *buf) { Eina_Strbuf *sbuf = eina_strbuf_new(); eina_strbuf_append_printf(sbuf, "TSUITE_VERBOSE=%d ", _verbose); eina_strbuf_append_printf(sbuf, "TSUITE_FILE_NAME='%s/%s.rec' ", _base_dir, ent->name); eina_strbuf_append_printf(sbuf, "TSUITE_TEST_NAME='%s' ", ent->name); switch (_mode) { case RUN_PLAY: { eina_strbuf_append(sbuf, "ELM_ENGINE='buffer' "); eina_strbuf_append_printf(sbuf, "TSUITE_DEST_DIR='%s/%s' ", _dest_dir, CURRENT_SUBDIR); break; } case RUN_INIT: { eina_strbuf_append(sbuf, "ELM_ENGINE='buffer' "); eina_strbuf_append_printf(sbuf, "TSUITE_DEST_DIR='%s/%s' ", _dest_dir, ORIG_SUBDIR); break; } default: break; } eina_strbuf_append_printf(sbuf, "LD_PRELOAD='%s/exactness/libexactness_%s.so' %s %s %s", PACKAGE_LIBDIR, "player", CONFIG, _wrap_command, ent->command); strncpy(buf, eina_strbuf_string_get(sbuf), SCHEDULER_CMD_SIZE-1); eina_strbuf_free(sbuf); _printf(1, "Command: %s\n", buf); } static Eina_Bool _check_prefix(const char *prefix, const char *name) { unsigned int len = strlen(prefix); return (!strncmp(name, prefix, len) && (strlen(name) > len) && (name[len] == SHOT_DELIMITER)); } static void _prefix_rm_cb(const char *name, const char *path, void *data) { const char *prefix = data; if (_check_prefix(prefix, name)) { char buf[EXACTNESS_PATH_MAX]; snprintf(buf, EXACTNESS_PATH_MAX, "%s/%s", path, name); if (unlink(buf)) { printf("Failed deleting '%s/%s': ", path, name); perror(""); } } } static void _run_test_prefix_rm(const char *dir, const char *prefix) { eina_file_dir_list(dir, 0, _prefix_rm_cb, (void *) prefix); } static Eina_Bool _job_deleted_cb(void *data EINA_UNUSED, int type EINA_UNUSED, void *event) { Ecore_Exe_Event_Del *msg = (Ecore_Exe_Event_Del *) event; if ((msg->exit_code != 0) || (msg->exit_signal != 0)) { List_Entry *ent = ecore_exe_data_get(msg->exe); _errors = eina_list_append(_errors, ent); } _running_jobs--; _job_consume(); /* If all jobs are done. */ if (!_running_jobs) { ecore_main_loop_quit(); return ECORE_CALLBACK_DONE; } return ECORE_CALLBACK_RENEW; } static Eina_Bool _job_consume() { static Ecore_Event_Handler *job_del_callback_handler = NULL; char buf[SCHEDULER_CMD_SIZE]; Ecore_Exe *exe; List_Entry *ent = _next_test_to_run; if (_running_jobs == _max_jobs) return EINA_FALSE; if (!ent) return EINA_FALSE; _running_jobs++; _tests_executed++; _run_command_prepare(ent, buf); switch (_mode) { case RUN_PLAY: { _run_test_prefix_rm(CURRENT_SUBDIR, ent->name); _printf(1, "Running %s\n", ent->name); break; } case RUN_INIT: { _run_test_prefix_rm(ORIG_SUBDIR, ent->name); break; } default: break; } if (!job_del_callback_handler) { job_del_callback_handler = ecore_event_handler_add(ECORE_EXE_EVENT_DEL, _job_deleted_cb, NULL); } exe = ecore_exe_pipe_run(buf, ECORE_EXE_TERM_WITH_PARENT, ent); _next_test_to_run = EINA_INLIST_CONTAINER_GET( EINA_INLIST_GET(ent)->next, List_Entry); if (!exe) { fprintf(stderr, "Failed executing test '%s'\n", ent->name); } return EINA_TRUE; } static void _scheduler_run() { while (_job_consume()); } static Eina_Bool _file_sha1_get(const char *filename, unsigned char *result) { Eina_File *f = NULL; const char *key = "0123456789abcde"; int key_len = strlen(key); unsigned int size = 0; Eina_Binbuf *buf = NULL; void *data = NULL; f = eina_file_open(filename, EINA_FALSE); if (!f) goto false; size = eina_file_size_get(f); if (size < 1) goto false; data = eina_file_map_all(f, EINA_FILE_POPULATE); if (!data) goto false; buf = eina_binbuf_manage_new(data, size, EINA_TRUE); if (!buf) { fprintf(stderr, "Could not create Binary Buffer"); goto false; } if (!emile_binbuf_hmac_sha1(key, key_len, buf, result)) { fprintf(stderr, "Cannot generate sha1 for image"); goto false; } eina_binbuf_free(buf); eina_file_close(f); return EINA_TRUE; false: if (buf) eina_binbuf_free(buf); if (f) eina_file_close(f); return EINA_FALSE; } #define _DIGEST_SIZE 20 static Eina_Bool _is_equal(const char *filename1, const char *filename2) { unsigned char res1[_DIGEST_SIZE], res2[_DIGEST_SIZE]; if (!_file_sha1_get(filename1, res1)) return EINA_FALSE; if (!_file_sha1_get(filename2, res2)) return EINA_FALSE; return !memcmp(res1, res2, _DIGEST_SIZE); } static void _compare_list_cb(const char *name, const char *path EINA_UNUSED, void *data) { const char *prefix = data; if (_check_prefix(prefix, name)) { char filename1[EXACTNESS_PATH_MAX], filename2[EXACTNESS_PATH_MAX]; snprintf(filename1, EXACTNESS_PATH_MAX, "%s/%s/%s", _dest_dir, ORIG_SUBDIR, name); snprintf(filename2, EXACTNESS_PATH_MAX, "%s/%s/%s", _dest_dir, CURRENT_SUBDIR, name); if (!_is_equal(filename1, filename2)) { char buf[EXACTNESS_PATH_MAX]; _compare_errors = eina_list_append(_compare_errors, strdup(name)); /* FIXME: Clean up. */ snprintf(buf, EXACTNESS_PATH_MAX, "compare '%s' '%s' '%s/%s/comp_%s'", filename1, filename2, _dest_dir, CURRENT_SUBDIR, name); if (system(buf)) { fprintf(stderr, "Failed image comparing '%s'\n", name); } } } } static void _run_test_compare(const List_Entry *ent) { char origdir[EXACTNESS_PATH_MAX]; snprintf(origdir, EXACTNESS_PATH_MAX, "%s/%s", _dest_dir, ORIG_SUBDIR); eina_file_dir_list(origdir, 0, _compare_list_cb, ent->name); } static List_Entry * _list_file_load(const char *filename) { List_Entry *ret = NULL; char buf[BUF_SIZE] = ""; FILE *file; file = fopen(filename, "r"); if (!file) { perror("Failed opening list file"); return NULL; } while (fgets(buf, BUF_SIZE, file)) { /* Skip comment/empty lines. */ if ((*buf == '#') || (*buf == '\n') || (!*buf)) continue; char *tmp; List_Entry *cur = calloc(1, sizeof(*cur)); cur->name = strdup(buf); /* Set the command to the second half and put a \0 in between. */ tmp = strchr(cur->name, ' '); if (tmp) { *tmp = '\0'; cur->command = tmp + 1; } else { /* FIXME: error. */ cur->command = ""; } /* Replace the newline char with a \0. */ tmp = strchr(cur->command, '\n'); if (tmp) { *tmp = '\0'; } ret = EINA_INLIST_CONTAINER_GET( eina_inlist_append(EINA_INLIST_GET(ret), EINA_INLIST_GET(cur)), List_Entry); } return ret; } static void _list_file_free(List_Entry *list) { while (list) { List_Entry *ent = list; list = EINA_INLIST_CONTAINER_GET(EINA_INLIST_GET(list)->next, List_Entry); free(ent->name); free(ent); /* we don't free ent->command because it's allocated together. */ } } static int _errors_sort_cb(List_Entry *a, List_Entry *b) { return strcmp(a->name, b->name); } static const Ecore_Getopt optdesc = { "exactness", "%prog [options] <-r|-p|-i|-s> ", PACKAGE_VERSION, "(C) 2013 Enlightenment", "BSD", "A pixel perfect test suite for EFL based applications.", 0, { ECORE_GETOPT_STORE_STR('b', "base-dir", "The location of the rec files."), ECORE_GETOPT_STORE_STR('d', "dest-dir", "The location of the images."), ECORE_GETOPT_STORE_STR('w', "wrap", "Use a custom command to launch the tests (e.g valgrind)."), ECORE_GETOPT_STORE_USHORT('j', "jobs", "The number of jobs to run in parallel."), ECORE_GETOPT_STORE_TRUE('p', "play", "Run in play mode."), ECORE_GETOPT_STORE_TRUE('i', "init", "Run in init mode."), ECORE_GETOPT_STORE_TRUE('s', "simulation", "Run in simulation mode."), ECORE_GETOPT_COUNT('v', "verbose", "Turn verbose messages on."), ECORE_GETOPT_LICENSE('L', "license"), ECORE_GETOPT_COPYRIGHT('C', "copyright"), ECORE_GETOPT_VERSION('V', "version"), ECORE_GETOPT_HELP('h', "help"), ECORE_GETOPT_SENTINEL } }; int main(int argc, char *argv[]) { int ret = 0; List_Entry *test_list; int args = 0; const char *list_file = ""; char tmp[EXACTNESS_PATH_MAX]; Eina_Bool mode_play = EINA_FALSE, mode_init = EINA_FALSE, mode_simulation = EINA_FALSE; Eina_Bool want_quit = EINA_FALSE; Ecore_Getopt_Value values[] = { ECORE_GETOPT_VALUE_STR(_base_dir), ECORE_GETOPT_VALUE_STR(_dest_dir), ECORE_GETOPT_VALUE_STR(_wrap_command), ECORE_GETOPT_VALUE_USHORT(_max_jobs), ECORE_GETOPT_VALUE_BOOL(mode_play), ECORE_GETOPT_VALUE_BOOL(mode_init), ECORE_GETOPT_VALUE_BOOL(mode_simulation), ECORE_GETOPT_VALUE_INT(_verbose), ECORE_GETOPT_VALUE_BOOL(want_quit), ECORE_GETOPT_VALUE_BOOL(want_quit), ECORE_GETOPT_VALUE_BOOL(want_quit), ECORE_GETOPT_VALUE_BOOL(want_quit), ECORE_GETOPT_VALUE_NONE }; ecore_init(); mode_play = mode_init = mode_simulation = EINA_FALSE; want_quit = EINA_FALSE; _base_dir = "./recordings"; _dest_dir = "./"; _wrap_command = ""; _max_jobs = 1; _verbose = 0; args = ecore_getopt_parse(&optdesc, values, argc, argv); if (args < 0) { fprintf(stderr, "Failed parsing arguments.\n"); ret = 1; goto end; } else if (want_quit) { ret = 1; goto end; } else if (args == argc) { fprintf(stderr, "Expected test list as the last argument..\n"); ecore_getopt_help(stderr, &optdesc); ret = 1; goto end; } else if (mode_play + mode_init + mode_simulation != 1) { fprintf(stderr, "At least and only one of the running modes can be set.\n"); ecore_getopt_help(stderr, &optdesc); ret = 1; goto end; } list_file = argv[args]; /* Load the list file and start iterating over the records. */ test_list = _list_file_load(list_file); _next_test_to_run = test_list; if (!test_list) { fprintf(stderr, "No matching tests found in '%s'\n", list_file); ret = 1; goto end; } /* Pre-run summary */ fprintf(stderr, "Running with settings:\n"); fprintf(stderr, "\tConcurrent jobs: %d\n", _max_jobs); fprintf(stderr, "\tTest list: %s\n", list_file); fprintf(stderr, "\tBase dir: %s\n", _base_dir); fprintf(stderr, "\tDest dir: %s\n", _dest_dir); if (mode_play) { _mode = RUN_PLAY; if (snprintf(tmp, EXACTNESS_PATH_MAX, "%s/%s", _dest_dir, CURRENT_SUBDIR) >= EXACTNESS_PATH_MAX) { fprintf(stderr, "Path too long: %s", tmp); ret = 1; goto end; } mkdir(tmp, 0744); } else if (mode_init) { _mode = RUN_INIT; if (snprintf(tmp, EXACTNESS_PATH_MAX, "%s/%s", _dest_dir, ORIG_SUBDIR) >= EXACTNESS_PATH_MAX) { fprintf(stderr, "Path too long: %s", tmp); ret = 1; goto end; } mkdir(tmp, 0744); } else if (mode_simulation) { _mode = RUN_SIMULATION; } _scheduler_run(); ecore_main_loop_begin(); /* Results */ printf("*******************************************************\n"); if (mode_play) { List_Entry *list_itr; EINA_INLIST_FOREACH(test_list, list_itr) { _run_test_compare(list_itr); } } printf("Finished executing %u out of %u tests.\n", _tests_executed, eina_inlist_count(EINA_INLIST_GET(test_list))); /* Sort the errors and the compare_errors. */ _errors = eina_list_sort(_errors, 0, (Eina_Compare_Cb) _errors_sort_cb); _compare_errors = eina_list_sort(_compare_errors, 0, (Eina_Compare_Cb) strcmp); if (_errors || _compare_errors) { FILE *report_file; char report_filename[EXACTNESS_PATH_MAX] = ""; /* Generate the filename. */ snprintf(report_filename, EXACTNESS_PATH_MAX, "%s/%s/errors.html", _dest_dir, mode_init ? ORIG_SUBDIR : CURRENT_SUBDIR); report_file = fopen(report_filename, "w+"); if (report_file) { printf("%s %p\n", report_filename, report_file); fprintf(report_file, "" "Exactness report"); if (_errors) { fprintf(report_file, "

Tests that failed execution:

    "); Eina_List *itr; List_Entry *ent; printf("List of tests that failed execution:\n"); EINA_LIST_FOREACH(_errors, itr, ent) { printf("\t* %s\n", ent->name); fprintf(report_file, "
  • %s
  • ", ent->name); } fprintf(report_file, "
"); } if (_compare_errors) { fprintf(report_file, "

Images that failed comparison: (Original, Current, Diff)

    "); char *test_name; printf("List of images that failed comparison:\n"); EINA_LIST_FREE(_compare_errors, test_name) { printf("\t* %s\n", test_name); fprintf(report_file, "
  • %s

    Original Current Diff
  • ", test_name, test_name, test_name, test_name); free(test_name); } fprintf(report_file, "
"); } fprintf(report_file, ""); printf("Report html: %s\n", report_filename); ret = 1; } else { perror("Failed opening report file"); } } _list_file_free(test_list); end: ecore_shutdown(); return ret; }