Made the exactness script a C program (with improvements).

It's now considerably faster thanks to concurrency.
It's cleaner (almost the same loc).
Better error output.
Easier to use.
devs/tasn/jenkins
Tom Hacohen 10 years ago
parent c3340d7909
commit c2547c0199
  1. 2
      README
  2. 3
      configure.ac
  3. 2
      src/Makefile.am
  4. 19
      src/bin/Makefile.am
  5. 185
      src/bin/exactness.c
  6. 4
      src/bin/exactness_config.c
  7. 34
      src/bin/exactness_config.h
  8. 72
      src/bin/list_file.c
  9. 17
      src/bin/list_file.h
  10. 295
      src/bin/md5/md5.c
  11. 45
      src/bin/md5/md5.h
  12. 146
      src/bin/run_test.c
  13. 14
      src/bin/run_test.h
  14. 94
      src/bin/scheduler.c
  15. 12
      src/bin/scheduler.h
  16. 10
      src/scripts/Makefile.am
  17. 537
      src/scripts/exactness.in

@ -1,3 +1,5 @@
POSSIBLY OUTDATED
exactness is a software package aimed to automate Elementary testing
after updating elm code.
The testing process is composed of running widget test,

@ -15,6 +15,8 @@ AC_PROG_CC
AC_HEADER_STDC
AC_C_CONST
AM_PROG_CC_C_O
PKG_PROG_PKG_CONFIG
AC_PROG_LIBTOOL
@ -41,6 +43,7 @@ PKG_CHECK_MODULES([EFL],
AC_OUTPUT([
Makefile
src/Makefile
src/bin/Makefile
src/lib/Makefile
src/scripts/Makefile
data/Makefile

@ -1,4 +1,4 @@
MAINTAINERCLEANFILES = Makefile.in
SUBDIRS = lib scripts
SUBDIRS = lib bin

@ -0,0 +1,19 @@
MAINTAINERCLEANFILES = Makefile.in
bin_PROGRAMS = exactness
exactness_SOURCES = \
exactness.c \
list_file.c \
exactness_config.c \
scheduler.c \
run_test.c \
md5/md5.c
exactness_LDADD = \
@EFL_LIBS@
exactness_CFLAGS = \
@EFL_CFLAGS@ \
-DPACKAGE_LIBDIR=\"$(libdir)\" \
-DPACKAGE_DATADIR=\"$(datadir)\"

@ -0,0 +1,185 @@
#include <Ecore.h>
#include <Ecore_Getopt.h>
#include "list_file.h"
#include "exactness_config.h"
#include "run_test.h"
#include "scheduler.h"
#include "config.h"
static const Ecore_Getopt optdesc = {
"exactness",
"%prog [options] <-r|-p|-i|-s> <list file>",
PACKAGE_VERSION,
"(C) 2013 Enlightenment",
"BSD",
"A pixel perfect test suite for EFL based applications.",
0,
{
ECORE_GETOPT_STORE_STR('b', "base-dir", "The location of the rec files."),
ECORE_GETOPT_STORE_STR('d', "dest-dir", "The location of the images."),
ECORE_GETOPT_STORE_USHORT('j', "jobs", "The number of jobs to run in parallel."),
ECORE_GETOPT_STORE_TRUE('r', "record", "Run in record mode."),
ECORE_GETOPT_STORE_TRUE('p', "play", "Run in play mode."),
ECORE_GETOPT_STORE_TRUE('i', "init", "Run in init mode."),
ECORE_GETOPT_STORE_TRUE('s', "simulation", "Run in simulation mode."),
ECORE_GETOPT_STORE_TRUE('v', "verbose", "Turn verbose messages on."),
ECORE_GETOPT_LICENSE('L', "license"),
ECORE_GETOPT_COPYRIGHT('C', "copyright"),
ECORE_GETOPT_VERSION('V', "version"),
ECORE_GETOPT_HELP('h', "help"),
ECORE_GETOPT_SENTINEL
}
};
int
main(int argc, char *argv[])
{
int ret = 0;
List_Entry *test_list;
int args = 0;
const char *list_file = "";
Eina_Bool mode_record, mode_play, mode_init, mode_simulation;
Eina_Bool want_quit;
Ecore_Getopt_Value values[] = {
ECORE_GETOPT_VALUE_STR(exactness_config.base_dir),
ECORE_GETOPT_VALUE_STR(exactness_config.dest_dir),
ECORE_GETOPT_VALUE_USHORT(exactness_config.jobs),
ECORE_GETOPT_VALUE_BOOL(mode_record),
ECORE_GETOPT_VALUE_BOOL(mode_play),
ECORE_GETOPT_VALUE_BOOL(mode_init),
ECORE_GETOPT_VALUE_BOOL(mode_simulation),
ECORE_GETOPT_VALUE_BOOL(exactness_config.verbose),
ECORE_GETOPT_VALUE_BOOL(want_quit),
ECORE_GETOPT_VALUE_BOOL(want_quit),
ECORE_GETOPT_VALUE_BOOL(want_quit),
ECORE_GETOPT_VALUE_BOOL(want_quit),
ECORE_GETOPT_VALUE_NONE
};
ecore_init();
mode_record = mode_play = mode_init = mode_simulation = EINA_FALSE;
want_quit = EINA_FALSE;
exactness_config.base_dir = PACKAGE_DATADIR "/exactness/recordings";
exactness_config.dest_dir = "./";
exactness_config.jobs = 1;
exactness_config.verbose = EINA_FALSE;
args = ecore_getopt_parse(&optdesc, values, argc, argv);
if (args < 0)
{
fprintf(stderr, "Failed parsing arguments.\n");
ret = 1;
goto end;
}
else if (want_quit)
{
ret = 1;
goto end;
}
else if (args == argc)
{
fprintf(stderr, "Expected test list as the last argument..\n");
ecore_getopt_help(stderr, &optdesc);
ret = 1;
goto end;
}
else if (mode_record + mode_play + mode_init + mode_simulation != 1)
{
fprintf(stderr, "At least and only one of the running modes can be set.\n");
ecore_getopt_help(stderr, &optdesc);
ret = 1;
goto end;
}
list_file = argv[args];
/* Load the list file and start iterating over the records. */
test_list = list_file_load(list_file);
if (!test_list)
{
fprintf(stderr, "No matching tests found in '%s'\n", list_file);
ret = 1;
goto end;
}
/* Pre-run summary */
fprintf(stderr, "Running with settings:\n");
fprintf(stderr, "\tConcurrent jobs: %d\n", exactness_config.jobs);
fprintf(stderr, "\tTest list: %s\n", list_file);
fprintf(stderr, "\tBase dir: %s\n", exactness_config.base_dir);
fprintf(stderr, "\tDest dir: %s\n", exactness_config.dest_dir);
if (mode_record)
{
scheduler_run(run_test_record, test_list);
}
else if (mode_play)
{
mkdir(CURRENT_SUBDIR, 0744);
scheduler_run(run_test_play, test_list);
}
else if (mode_init)
{
mkdir(ORIG_SUBDIR, 0744);
scheduler_run(run_test_init, test_list);
}
else if (mode_simulation)
{
scheduler_run(run_test_simulation, test_list);
}
ecore_main_loop_begin();
/* Results */
printf("*******************************************************\n");
if (mode_play)
{
List_Entry *list_itr;
EINA_INLIST_FOREACH(test_list, list_itr)
{
run_test_compare(list_itr);
}
}
printf("Finished executing %u out of %u tests.\n",
exactness_ctx.tests_executed,
eina_inlist_count(EINA_INLIST_GET(test_list)));
if (exactness_ctx.errors)
{
Eina_List *itr;
List_Entry *ent;
printf("List of tests that failed execution:\n");
EINA_LIST_FOREACH(exactness_ctx.errors, itr, ent)
{
printf("\t* %s\n", ent->name);
}
ret = 1;
}
if (exactness_ctx.compare_errors)
{
char *test_name;
printf("List of images that failed comparison:\n");
EINA_LIST_FREE(exactness_ctx.compare_errors, test_name)
{
printf("\t* %s\n", test_name);
free(test_name);
}
ret = 1;
}
list_file_free(test_list);
end:
ecore_shutdown();
return ret;
}

@ -0,0 +1,4 @@
#include "exactness_config.h"
Exactness_Config exactness_config;
Exactness_Ctx exactness_ctx;

@ -0,0 +1,34 @@
#ifndef EXACTNESS_CONFIG_H
#define EXACTNESS_CONFIG_H
#include <Eina.h>
typedef struct _Exactness_Config Exactness_Config;
struct _Exactness_Config
{
unsigned short jobs;
char *base_dir;
char *dest_dir;
Eina_Bool verbose;
};
extern Exactness_Config exactness_config;
typedef struct _Exactness_Ctx Exactness_Ctx;
struct _Exactness_Ctx
{
unsigned int tests_executed;
Eina_List *errors;
Eina_List *compare_errors;
};
extern Exactness_Ctx exactness_ctx;
#define ORIG_SUBDIR "orig"
#define CURRENT_SUBDIR "current"
#define EXACTNESS_PATH_MAX 1024
#endif

@ -0,0 +1,72 @@
#include <stdio.h>
#include "list_file.h"
#define BUF_SIZE 1024
List_Entry *
list_file_load(const char *filename)
{
List_Entry *ret = NULL;
char buf[BUF_SIZE] = "";
FILE *file;
file = fopen(filename, "r");
if (!file)
{
perror("Failed opening list file");
return NULL;
}
while (fgets(buf, BUF_SIZE, file))
{
/* Skip comment/empty lines. */
if ((*buf == '#') || (*buf == '\n') || (!*buf))
continue;
char *tmp;
List_Entry *cur = calloc(1, sizeof(*cur));
cur->name = strdup(buf);
/* Set the command to the second half and put a \0 in between. */
tmp = strchr(cur->name, ' ');
if (tmp)
{
*tmp = '\0';
cur->command = tmp + 1;
}
else
{
/* FIXME: error. */
cur->command = "";
}
/* Replace the newline char with a \0. */
tmp = strchr(cur->command, '\n');
if (tmp)
{
*tmp = '\0';
}
ret = EINA_INLIST_CONTAINER_GET(
eina_inlist_append(EINA_INLIST_GET(ret), EINA_INLIST_GET(cur)),
List_Entry);
}
return ret;
}
void
list_file_free(List_Entry *list)
{
while (list)
{
List_Entry *ent = list;
list = EINA_INLIST_CONTAINER_GET(EINA_INLIST_GET(list)->next,
List_Entry);
free(ent->name);
free(ent);
/* we don't free ent->command because it's allocated together. */
}
}

@ -0,0 +1,17 @@
#ifndef LIST_FILE_H
#define LIST_FILE_H
#include <Eina.h>
typedef struct _List_Entry List_Entry;
struct _List_Entry
{
EINA_INLIST;
char *name;
const char *command;
};
List_Entry *list_file_load(const char *filename);
void list_file_free(List_Entry *list);
#endif

@ -0,0 +1,295 @@
/*
* This is an OpenSSL-compatible implementation of the RSA Data Security, Inc.
* MD5 Message-Digest Algorithm (RFC 1321).
*
* Homepage:
* http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5
*
* Author:
* Alexander Peslyak, better known as Solar Designer <solar at openwall.com>
*
* This software was written by Alexander Peslyak in 2001. No copyright is
* claimed, and the software is hereby placed in the public domain.
* In case this attempt to disclaim copyright and place the software in the
* public domain is deemed null and void, then the software is
* Copyright (c) 2001 Alexander Peslyak and it is hereby released to the
* general public under the following terms:
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted.
*
* There's ABSOLUTELY NO WARRANTY, express or implied.
*
* (This is a heavily cut-down "BSD license".)
*
* This differs from Colin Plumb's older public domain implementation in that
* no exactly 32-bit integer data type is required (any 32-bit or wider
* unsigned integer data type will do), there's no compile-time endianness
* configuration, and the function prototypes match OpenSSL's. No code from
* Colin Plumb's implementation has been reused; this comment merely compares
* the properties of the two independent implementations.
*
* The primary goals of this implementation are portability and ease of use.
* It is meant to be fast, but not as fast as possible. Some known
* optimizations are not included to reduce source code size and avoid
* compile-time configuration.
*/
#ifndef HAVE_OPENSSL
#include <string.h>
#include "md5.h"
/*
* The basic MD5 functions.
*
* F and G are optimized compared to their RFC 1321 definitions for
* architectures that lack an AND-NOT instruction, just like in Colin Plumb's
* implementation.
*/
#define F(x, y, z) ((z) ^ ((x) & ((y) ^ (z))))
#define G(x, y, z) ((y) ^ ((z) & ((x) ^ (y))))
#define H(x, y, z) ((x) ^ (y) ^ (z))
#define I(x, y, z) ((y) ^ ((x) | ~(z)))
/*
* The MD5 transformation for all four rounds.
*/
#define STEP(f, a, b, c, d, x, t, s) \
(a) += f((b), (c), (d)) + (x) + (t); \
(a) = (((a) << (s)) | (((a) & 0xffffffff) >> (32 - (s)))); \
(a) += (b);
/*
* SET reads 4 input bytes in little-endian byte order and stores them
* in a properly aligned word in host byte order.
*
* The check for little-endian architectures that tolerate unaligned
* memory accesses is just an optimization. Nothing will break if it
* doesn't work.
*/
#if defined(__i386__) || defined(__x86_64__) || defined(__vax__)
#define SET(n) \
(*(MD5_u32plus *)&ptr[(n) * 4])
#define GET(n) \
SET(n)
#else
#define SET(n) \
(ctx->block[(n)] = \
(MD5_u32plus)ptr[(n) * 4] | \
((MD5_u32plus)ptr[(n) * 4 + 1] << 8) | \
((MD5_u32plus)ptr[(n) * 4 + 2] << 16) | \
((MD5_u32plus)ptr[(n) * 4 + 3] << 24))
#define GET(n) \
(ctx->block[(n)])
#endif
/*
* This processes one or more 64-byte data blocks, but does NOT update
* the bit counters. There are no alignment requirements.
*/
static void *body(MD5_CTX *ctx, void *data, unsigned long size)
{
unsigned char *ptr;
MD5_u32plus a, b, c, d;
MD5_u32plus saved_a, saved_b, saved_c, saved_d;
ptr = data;
a = ctx->a;
b = ctx->b;
c = ctx->c;
d = ctx->d;
do {
saved_a = a;
saved_b = b;
saved_c = c;
saved_d = d;
/* Round 1 */
STEP(F, a, b, c, d, SET(0), 0xd76aa478, 7)
STEP(F, d, a, b, c, SET(1), 0xe8c7b756, 12)
STEP(F, c, d, a, b, SET(2), 0x242070db, 17)
STEP(F, b, c, d, a, SET(3), 0xc1bdceee, 22)
STEP(F, a, b, c, d, SET(4), 0xf57c0faf, 7)
STEP(F, d, a, b, c, SET(5), 0x4787c62a, 12)
STEP(F, c, d, a, b, SET(6), 0xa8304613, 17)
STEP(F, b, c, d, a, SET(7), 0xfd469501, 22)
STEP(F, a, b, c, d, SET(8), 0x698098d8, 7)
STEP(F, d, a, b, c, SET(9), 0x8b44f7af, 12)
STEP(F, c, d, a, b, SET(10), 0xffff5bb1, 17)
STEP(F, b, c, d, a, SET(11), 0x895cd7be, 22)
STEP(F, a, b, c, d, SET(12), 0x6b901122, 7)
STEP(F, d, a, b, c, SET(13), 0xfd987193, 12)
STEP(F, c, d, a, b, SET(14), 0xa679438e, 17)
STEP(F, b, c, d, a, SET(15), 0x49b40821, 22)
/* Round 2 */
STEP(G, a, b, c, d, GET(1), 0xf61e2562, 5)
STEP(G, d, a, b, c, GET(6), 0xc040b340, 9)
STEP(G, c, d, a, b, GET(11), 0x265e5a51, 14)
STEP(G, b, c, d, a, GET(0), 0xe9b6c7aa, 20)
STEP(G, a, b, c, d, GET(5), 0xd62f105d, 5)
STEP(G, d, a, b, c, GET(10), 0x02441453, 9)
STEP(G, c, d, a, b, GET(15), 0xd8a1e681, 14)
STEP(G, b, c, d, a, GET(4), 0xe7d3fbc8, 20)
STEP(G, a, b, c, d, GET(9), 0x21e1cde6, 5)
STEP(G, d, a, b, c, GET(14), 0xc33707d6, 9)
STEP(G, c, d, a, b, GET(3), 0xf4d50d87, 14)
STEP(G, b, c, d, a, GET(8), 0x455a14ed, 20)
STEP(G, a, b, c, d, GET(13), 0xa9e3e905, 5)
STEP(G, d, a, b, c, GET(2), 0xfcefa3f8, 9)
STEP(G, c, d, a, b, GET(7), 0x676f02d9, 14)
STEP(G, b, c, d, a, GET(12), 0x8d2a4c8a, 20)
/* Round 3 */
STEP(H, a, b, c, d, GET(5), 0xfffa3942, 4)
STEP(H, d, a, b, c, GET(8), 0x8771f681, 11)
STEP(H, c, d, a, b, GET(11), 0x6d9d6122, 16)
STEP(H, b, c, d, a, GET(14), 0xfde5380c, 23)
STEP(H, a, b, c, d, GET(1), 0xa4beea44, 4)
STEP(H, d, a, b, c, GET(4), 0x4bdecfa9, 11)
STEP(H, c, d, a, b, GET(7), 0xf6bb4b60, 16)
STEP(H, b, c, d, a, GET(10), 0xbebfbc70, 23)
STEP(H, a, b, c, d, GET(13), 0x289b7ec6, 4)
STEP(H, d, a, b, c, GET(0), 0xeaa127fa, 11)
STEP(H, c, d, a, b, GET(3), 0xd4ef3085, 16)
STEP(H, b, c, d, a, GET(6), 0x04881d05, 23)
STEP(H, a, b, c, d, GET(9), 0xd9d4d039, 4)
STEP(H, d, a, b, c, GET(12), 0xe6db99e5, 11)
STEP(H, c, d, a, b, GET(15), 0x1fa27cf8, 16)
STEP(H, b, c, d, a, GET(2), 0xc4ac5665, 23)
/* Round 4 */
STEP(I, a, b, c, d, GET(0), 0xf4292244, 6)
STEP(I, d, a, b, c, GET(7), 0x432aff97, 10)
STEP(I, c, d, a, b, GET(14), 0xab9423a7, 15)
STEP(I, b, c, d, a, GET(5), 0xfc93a039, 21)
STEP(I, a, b, c, d, GET(12), 0x655b59c3, 6)
STEP(I, d, a, b, c, GET(3), 0x8f0ccc92, 10)
STEP(I, c, d, a, b, GET(10), 0xffeff47d, 15)
STEP(I, b, c, d, a, GET(1), 0x85845dd1, 21)
STEP(I, a, b, c, d, GET(8), 0x6fa87e4f, 6)
STEP(I, d, a, b, c, GET(15), 0xfe2ce6e0, 10)
STEP(I, c, d, a, b, GET(6), 0xa3014314, 15)
STEP(I, b, c, d, a, GET(13), 0x4e0811a1, 21)
STEP(I, a, b, c, d, GET(4), 0xf7537e82, 6)
STEP(I, d, a, b, c, GET(11), 0xbd3af235, 10)
STEP(I, c, d, a, b, GET(2), 0x2ad7d2bb, 15)
STEP(I, b, c, d, a, GET(9), 0xeb86d391, 21)
a += saved_a;
b += saved_b;
c += saved_c;
d += saved_d;
ptr += 64;
} while (size -= 64);
ctx->a = a;
ctx->b = b;
ctx->c = c;
ctx->d = d;
return ptr;
}
void MD5_Init(MD5_CTX *ctx)
{
ctx->a = 0x67452301;
ctx->b = 0xefcdab89;
ctx->c = 0x98badcfe;
ctx->d = 0x10325476;
ctx->lo = 0;
ctx->hi = 0;
}
void MD5_Update(MD5_CTX *ctx, void *data, unsigned long size)
{
MD5_u32plus saved_lo;
unsigned long used, free;
saved_lo = ctx->lo;
if ((ctx->lo = (saved_lo + size) & 0x1fffffff) < saved_lo)
ctx->hi++;
ctx->hi += size >> 29;
used = saved_lo & 0x3f;
if (used) {
free = 64 - used;
if (size < free) {
memcpy(&ctx->buffer[used], data, size);
return;
}
memcpy(&ctx->buffer[used], data, free);
data = (unsigned char *)data + free;
size -= free;
body(ctx, ctx->buffer, 64);
}
if (size >= 64) {
data = body(ctx, data, size & ~(unsigned long)0x3f);
size &= 0x3f;
}
memcpy(ctx->buffer, data, size);
}
void MD5_Final(unsigned char *result, MD5_CTX *ctx)
{
unsigned long used, free;
used = ctx->lo & 0x3f;
ctx->buffer[used++] = 0x80;
free = 64 - used;
if (free < 8) {
memset(&ctx->buffer[used], 0, free);
body(ctx, ctx->buffer, 64);
used = 0;
free = 64;
}
memset(&ctx->buffer[used], 0, free - 8);
ctx->lo <<= 3;
ctx->buffer[56] = ctx->lo;
ctx->buffer[57] = ctx->lo >> 8;
ctx->buffer[58] = ctx->lo >> 16;
ctx->buffer[59] = ctx->lo >> 24;
ctx->buffer[60] = ctx->hi;
ctx->buffer[61] = ctx->hi >> 8;
ctx->buffer[62] = ctx->hi >> 16;
ctx->buffer[63] = ctx->hi >> 24;
body(ctx, ctx->buffer, 64);
result[0] = ctx->a;
result[1] = ctx->a >> 8;
result[2] = ctx->a >> 16;
result[3] = ctx->a >> 24;
result[4] = ctx->b;
result[5] = ctx->b >> 8;
result[6] = ctx->b >> 16;
result[7] = ctx->b >> 24;
result[8] = ctx->c;
result[9] = ctx->c >> 8;
result[10] = ctx->c >> 16;
result[11] = ctx->c >> 24;
result[12] = ctx->d;
result[13] = ctx->d >> 8;
result[14] = ctx->d >> 16;
result[15] = ctx->d >> 24;
memset(ctx, 0, sizeof(*ctx));
}
#endif

@ -0,0 +1,45 @@
/*
* This is an OpenSSL-compatible implementation of the RSA Data Security, Inc.
* MD5 Message-Digest Algorithm (RFC 1321).
*
* Homepage:
* http://openwall.info/wiki/people/solar/software/public-domain-source-code/md5
*
* Author:
* Alexander Peslyak, better known as Solar Designer <solar at openwall.com>
*
* This software was written by Alexander Peslyak in 2001. No copyright is
* claimed, and the software is hereby placed in the public domain.
* In case this attempt to disclaim copyright and place the software in the
* public domain is deemed null and void, then the software is
* Copyright (c) 2001 Alexander Peslyak and it is hereby released to the
* general public under the following terms:
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted.
*
* There's ABSOLUTELY NO WARRANTY, express or implied.
*
* See md5.c for more information.
*/
#ifdef HAVE_OPENSSL
#include <openssl/md5.h>
#elif !defined(_MD5_H)
#define _MD5_H
/* Any 32-bit or wider unsigned integer data type will do */
typedef unsigned int MD5_u32plus;
typedef struct {
MD5_u32plus lo, hi;
MD5_u32plus a, b, c, d;
unsigned char buffer[64];
MD5_u32plus block[16];
} MD5_CTX;
extern void MD5_Init(MD5_CTX *ctx);
extern void MD5_Update(MD5_CTX *ctx, void *data, unsigned long size);
extern void MD5_Final(unsigned char *result, MD5_CTX *ctx);
#endif

@ -0,0 +1,146 @@
#include <stdio.h>
#include <unistd.h>
#include "config.h"
#include "scheduler.h"
#include "run_test.h"
#include "list_file.h"
#include "exactness_config.h"
#include "md5/md5.h"
#define LIBEXACTNESS_PATH PACKAGE_LIBDIR "/exactness/libexactness.so"
void
run_test_simulation(const List_Entry *ent, char *buf)
{
snprintf(buf, SCHEDULER_CMD_SIZE, "TSUITE_DEST_DIR='%s' TSUITE_FILE_NAME='%s/%s.rec' TSUITE_TEST_NAME='%s' LD_PRELOAD='%s' %s",
exactness_config.dest_dir,
exactness_config.base_dir, ent->name,
ent->name, LIBEXACTNESS_PATH,
ent->command);
}
void
run_test_play(const List_Entry *ent, char *buf)
{
snprintf(buf, SCHEDULER_CMD_SIZE, "ELM_ENGINE='buffer' TSUITE_DEST_DIR='%s/" CURRENT_SUBDIR "' TSUITE_FILE_NAME='%s/%s.rec' TSUITE_TEST_NAME='%s' LD_PRELOAD='%s' %s",
exactness_config.dest_dir,
exactness_config.base_dir, ent->name,
ent->name, LIBEXACTNESS_PATH,
ent->command);
run_test_prefix_rm(CURRENT_SUBDIR, ent->name);
}
void
run_test_record(const List_Entry *ent, char *buf)
{
snprintf(buf, SCHEDULER_CMD_SIZE, "TSUITE_RECORDING='rec' TSUITE_DEST_DIR='%s' TSUITE_FILE_NAME='%s/%s.rec' TSUITE_TEST_NAME='%s' LD_PRELOAD='%s' %s",
exactness_config.dest_dir,
exactness_config.base_dir, ent->name,
ent->name, LIBEXACTNESS_PATH,
ent->command);
}
void
run_test_init(const List_Entry *ent, char *buf)
{
snprintf(buf, SCHEDULER_CMD_SIZE, "ELM_ENGINE='buffer' TSUITE_DEST_DIR='%s/" ORIG_SUBDIR "' TSUITE_FILE_NAME='%s/%s.rec' TSUITE_TEST_NAME='%s' LD_PRELOAD='%s' %s",
exactness_config.dest_dir,
exactness_config.base_dir, ent->name,
ent->name, LIBEXACTNESS_PATH,
ent->command);
run_test_prefix_rm(ORIG_SUBDIR, ent->name);
}
static Eina_Bool
_file_md5_get(const char *filename, unsigned char *result)
{
MD5_CTX ctx;
Eina_File *file;
file = eina_file_open(filename, 0);
if (!file)
return EINA_FALSE;
MD5_Init(&ctx);
MD5_Update(&ctx, eina_file_map_all(file, EINA_FILE_SEQUENTIAL), eina_file_size_get(file));
MD5_Final(result, &ctx);
eina_file_close(file);
return EINA_TRUE;
}
#define _MD5_SIZE 16
static Eina_Bool
_md5_is_equal(const char *filename1, const char *filename2)
{
unsigned char res1[_MD5_SIZE], res2[_MD5_SIZE];
if (!_file_md5_get(filename1, res1))
return EINA_FALSE;
if (!_file_md5_get(filename2, res2))
return EINA_FALSE;
return !memcmp(res1, res2, _MD5_SIZE);
}
static void
_compare_list_cb(const char *name, const char *path EINA_UNUSED, void *data)
{
const char *prefix = data;
if (!strncmp(name, prefix, strlen(prefix)))
{
char filename1[EXACTNESS_PATH_MAX], filename2[EXACTNESS_PATH_MAX];
snprintf(filename1, EXACTNESS_PATH_MAX, "%s/%s", CURRENT_SUBDIR, name);
snprintf(filename2, EXACTNESS_PATH_MAX, "%s/%s", ORIG_SUBDIR, name);
if (!_md5_is_equal(filename1, filename2))
{
char buf[EXACTNESS_PATH_MAX];
exactness_ctx.compare_errors =
eina_list_append(exactness_ctx.compare_errors,
strdup(name));
/* FIXME: Clean up. */
snprintf(buf, EXACTNESS_PATH_MAX,
"compare '%s/%s' '%s/%s' '%s/comp_%s'",
ORIG_SUBDIR, name,
CURRENT_SUBDIR, name,
CURRENT_SUBDIR, name);
if (system(buf))
{
fprintf(stderr, "Failed image comparing '%s'\n", name);
}
}
}
}
void
run_test_compare(const List_Entry *ent)
{
eina_file_dir_list(ORIG_SUBDIR, 0, _compare_list_cb, ent->name);
}
static void
_prefix_rm_cb(const char *name, const char *path, void *data)
{
const char *prefix = data;
if (!strncmp(name, prefix, strlen(prefix)))
{
char buf[EXACTNESS_PATH_MAX];
snprintf(buf, EXACTNESS_PATH_MAX, "%s/%s", path, name);
if (unlink(buf))
{
printf("Failed deleting '%s/%s': ", path, name);
perror("");
}
}
}
void
run_test_prefix_rm(const char *dir, const char *prefix)
{
eina_file_dir_list(dir, 0, _prefix_rm_cb, (void *) prefix);
}

@ -0,0 +1,14 @@
#ifndef RUN_TEST_H
#define RUN_TEST_H
#include "list_file.h"
void run_test_simulation(const List_Entry *ent, char *buf);
void run_test_play(const List_Entry *ent, char *buf);
void run_test_record(const List_Entry *ent, char *buf);
void run_test_init(const List_Entry *ent, char *buf);
void run_test_compare(const List_Entry *ent);
void run_test_prefix_rm(const char *dir, const char *prefix);
#endif

@ -0,0 +1,94 @@
#include <Ecore.h>
#include "scheduler.h"
#include "exactness_config.h"
typedef struct
{
Scheduler_Cb prepare_func;
List_Entry *last;
unsigned short jobs;
} Scheduler_Ctx;
static Ecore_Event_Handler *_job_del_callback_handler = NULL;
static Eina_Bool _job_dispatch(List_Entry *ent, Scheduler_Ctx *ctx);
static Eina_Bool
_job_deleted_cb(void *data, int type EINA_UNUSED, void *event)
{
Ecore_Exe_Event_Del *msg = (Ecore_Exe_Event_Del *) event;
Scheduler_Ctx *ctx = data;
if (msg->exit_code != 0)
{
List_Entry *ent = ecore_exe_data_get(msg->exe);
exactness_ctx.errors = eina_list_append(exactness_ctx.errors, ent);
}
ctx->jobs++;
exactness_ctx.tests_executed++;
if (ctx->last && EINA_INLIST_GET(ctx->last)->next)
{
ctx->last = EINA_INLIST_CONTAINER_GET(
EINA_INLIST_GET(ctx->last)->next, List_Entry);
_job_dispatch(ctx->last, ctx);
}
/* If all jobs are done. */
if (ctx->jobs == exactness_config.jobs)
{
free(ctx);
ecore_main_loop_quit();
return ECORE_CALLBACK_DONE;
}
return ECORE_CALLBACK_RENEW;
}
static Eina_Bool
_job_dispatch(List_Entry *ent, Scheduler_Ctx *ctx)
{
char buf[SCHEDULER_CMD_SIZE];
Ecore_Exe *exe;
if (ctx->jobs == 0)
return EINA_FALSE;
ctx->jobs--;
ctx->prepare_func(ent, buf);
if (!_job_del_callback_handler)
{
_job_del_callback_handler = ecore_event_handler_add(ECORE_EXE_EVENT_DEL,
_job_deleted_cb, ctx);
}
exe = ecore_exe_pipe_run(buf, ECORE_EXE_TERM_WITH_PARENT, ent);
if (!exe)
{
fprintf(stderr, "Failed executing test '%s'\n", ent->name);
}
return EINA_TRUE;
}
void
scheduler_run(Scheduler_Cb prepare_func, List_Entry *list)
{
Scheduler_Ctx *ctx = calloc(1, sizeof(*ctx));
List_Entry *list_itr;
ctx->jobs = exactness_config.jobs;
ctx->prepare_func = prepare_func;
EINA_INLIST_FOREACH(list, list_itr)
{
if (!_job_dispatch(list_itr, ctx))
break;
ctx->last = list_itr;
}
}

@ -0,0 +1,12 @@
#ifndef SCHEDULER_H
#define SCHEDULER_H
#include "list_file.h"
#define SCHEDULER_CMD_SIZE 1024
typedef void (*Scheduler_Cb)(const List_Entry *, char *);
void scheduler_run(Scheduler_Cb prepare_func, List_Entry *list);
#endif

@ -1,10 +0,0 @@
MAINTAINERCLEANFILES = Makefile.in
exactness: exactness.in
$(AM_V_GEN) $(SED) -e "s|\@libdir\@|$(libdir)|" -e "s|\@datadir\@|$(datadir)|" $(srcdir)/exactness.in > $(builddir)/exactness
bin_SCRIPTS = exactness
CLEAN_FILES = exactness
EXTRA_DIST = exactness.in

@ -1,537 +0,0 @@
#!/bin/bash
# tsuite_script.sh -i this makes new 'orig' folder
# tsuite_script.sh -i -b [BaseDir] TestName1 [TestName2 ...] rewrite files for selcted tests in 'orig' folder
# tsuite_script.sh -r -b [BaseDir] [TestName1 TestName2 ...] ; this means record [all tests or TestName additional arg]
# tsuite_script.sh -b [BaseDir] -d FolderName -p [TestName1 TestName2 ...] this means play a record and put screenshots FolderName
# tsuite_script.sh -d FolderName -c [TestName1 TestName2 ...] this means compare "orig" with screenshots in FolderName
# When omitting DestDir we will use 'current' as default DestDir
#_DEBUG="on"
function DEBUG()
{
[ "$_DEBUG" == "on" ] && $@
}
function VERBOSE()
{
if $_verbose ; then
$@
fi
}
do_help () {
echo "Use $0 to test application screen-layout."
echo "First, you need to compose a tests file as follows:"
echo "Each line begins with test name"
echo "second field is test-command and [optional] params."
echo "Any line starting with '#' is a comment (ignored):"
echo
echo "# This is a comment line"
echo "TestName TestCmd [param1] [param2]"
echo
echo "Later, you run $0 with the tests file as parameter."
echo
echo "By default, exactness runs through test file running all tests specified."
echo "You may run selected tests by adding test name as param to exactness."
echo "Usage:"
echo "$0 -s TestsFile TestName1 [TestName2] [...]"
echo "Use this option to run selected tests without modifying your test file."
echo "TestName param has to match test name given in tests file (1st field)"
echo
echo
echo "Two additional parameters that $0 accepts:"
echo "BaseDir - This is where '.rec' files reside."
echo "DestDir - Where $0 creates test screen shots."
echo " Gets 'current' under 'pwd' by default ('orig' on init)"
echo
echo
echo "Use the following options:"
echo "To record tests:"
echo "$0 -r [-b BaseDir] TestsFile"
echo "Use BaseDir arg to create record files in specific folder."
echo "Otherwise pwd is used."
echo
echo "Pressing F2 while recording, sets screen shot at this stage of test."
echo "You may define env-var 'TSUITE_SHOT_KEY' to alter shot-key."
echo "'.rec' file is produced for each test in your TestsFile."
echo "File name is defined as 'TestName.rec' for each test."
echo
echo "You may test your record files with simulate option:"
echo "$0 -s [-b BaseDir] TestsFile"
echo
echo "You need to run $0 with init option prior"
echo "to using play option."
echo "Later, when doing play, PNG files are compared with"
echo "PNG files reside in 'orig' folder create when init."
echo
echo "To use init option:"
echo "$0 -i [-b BaseDir] TestsFile"
echo "Do not use DestDir param with init, target always 'orig'."
echo
echo "Use Play tests option to produce PNG files of screen shot:"
echo "$0 -p [-b BaseDir] [-d DestDir] TestsFile"
echo "Play option produces PNG files in DestDir."
echo "These are compares with PNGs in 'orig'."
echo "(created in 'init' phase)"
echo
echo "Use -v option for detailed flow-report."
echo "Thus, when running many tests, the output format makes"
echo "it easy to match output to a running test."
echo "Example:"
echo "$0 -v -p [-b BaseDir] [-d DestDir] TestsFile"
echo
echo "-b = Set the base dir, the location of the rec files."
echo "-p = Run in compare mode. Generates the current images and compares with the old ones."
echo "-r = Run in record mode. Record the '.rec' files."
echo "-i = Run in init mode. Generate the reference images."
echo "-s = Run in simulation mode. Similar to -p, but runs in visible window to show you what is actually recorded and does not actually generate images."
echo "-v = Run in verbose mode."
echo "-h/? = This message."
}
get_test_params () {
# This function analyze input line and sets test-file-name, rec-file-name
# reset globals
_test_name=
_test_cmd=
local line="$1"
local c=${line:0:1}
if [ "$c" = "#" ]
then
return 1
fi
local p=`expr index "$line" \ `
if [ $p -ne 0 ]
then
(( p-- ))
fi
_test_name=${line:0:p}
(( p++ ))
_test_cmd=${line:p}
# Test that input is valid
if [ -z "$_test_name" ]
then
_test_name=
_test_cmd=
return 1
fi
if [ -z "$_test_cmd" ]
then
_test_name=
_test_cmd=
return 1
fi
DEBUG echo test name=\""$_test_name"\"
DEBUG echo test cmd=\""$_test_cmd"\"
return 0
}
do_record () {
DEBUG printf "do_record()\n"
# This will run record for all test if no specific test specified
# or run recording of specified tests (names passed as parameter).
# run ALL tests to record
DEBUG echo do_record "$*"
get_test_params "$1"
if [ $? -ne 0 ]
then
return 0
fi
VERBOSE echo "do_record: $_test_name"
TSUITE_RECORDING='rec' TSUITE_DEST_DIR=${_dest_dir} TSUITE_FILE_NAME=${_base_dir}/${_test_name}.rec TSUITE_TEST_NAME=${_test_name} LD_PRELOAD=${OUR_LIBPATH}/libexactness.so eval ${_test_cmd}
}
do_simulation () {
# This will play simulation
# this will NOT produce screenshots
DEBUG echo do_simulation "$*"
get_test_params "$1"
if [ $? -ne 0 ]
then
return 0
fi
local file_name=${_base_dir}/${_test_name}.rec
if [ ! -e "$file_name" ]
then
echo Rec file "$file_name" not found.
return 1
fi
VERBOSE echo "do_simulation: $_test_name"
TSUITE_DEST_DIR=${_dest_dir} TSUITE_FILE_NAME=${file_name} TSUITE_TEST_NAME=${_test_name} LD_PRELOAD=${OUR_LIBPATH}/libexactness.so eval ${_test_cmd}
}
do_play () {
# This will play record for all test if specified.
# or play record of tests specified as parameter.
# run ALL tests to record
DEBUG echo base dir: "$_base_dir"
DEBUG echo dest dir: "$_dest_dir"
DEBUG echo do_play "$_dest_dir" "$*"
# Play recorded tests and produce PNG files.
# this will produce screenshots in "_dest_dir" folder
get_test_params "$1"
if [ $? -ne 0 ]
then
return 0
fi
local file_name=${_base_dir}/${_test_name}.rec
if [ ! -e "$file_name" ]
then
echo Rec file "$file_name" not found.
return 1
fi
if [ -e "$_dest_dir" ]
then
# Remove PNG files according to tests played
rm "$_dest_dir"/${_test_name}_[0-9]*.png &> /dev/null
else
# Create dest dir
mkdir -p "$_dest_dir" &> /dev/null
fi
VERBOSE echo "do_play: $_test_name"
ELM_ENGINE="buffer" TSUITE_DEST_DIR=${_dest_dir} TSUITE_FILE_NAME=${file_name} TSUITE_TEST_NAME=${_test_name} LD_PRELOAD=${OUR_LIBPATH}/libexactness.so eval ${_test_cmd}
}
compare_files () {
VERBOSE echo "compare_files: <$1> and <$2>"
if [ -e "$1" ]
# First file exists
then
local md_file1=`md5sum $1`
if [ -e "$2" ]
then
# Second file exists
local md_file2=`md5sum $2`
# Get md5 of both files
local md1=`echo "$md_file1" | cut -d ' ' -f1`
local md2=`echo "$md_file2" | cut -d ' ' -f1`
# Increase counter of comparisons
(( ncomp++ ))
# Compare md5 of both files
if [ "x$md1" != "x$md2" ]
then
if [ $comp_unavail -eq 0 ]
then
# Create diff-file with 'comp_' prefix.
local name=`basename "$1"`
compare "$1" "$2" "$_dest_dir"/comp_"$name"
else
echo "$name does not match."
fi
# Increment counter of files not identical.
(( nfail++ ))
fi
else
# Failed to find second file
echo "Test file was not found $2"
(( nerr++ ))
fi
else
# Failed to find first file
echo "Test file was not found $1"
(( nerr++ ))
fi
}
process_compare () {
# Process all files listed in array (param)
local files_list=( "$@" )
for line in "${files_list[@]}"
do
local name=`basename "$line"`
DEBUG echo "comparing $name"
compare_files "$_orig_dir"/"$name" "$_dest_dir"/"$name"
done
}
do_compare () {
DEBUG printf "do_compare()\n"
DEBUG echo orig dir: "$_orig_dir"
DEBUG echo dest dir: "$_dest_dir"
get_test_params "$1"
if [ $? -ne 0 ]; then
return 0;
fi
# This will compare files in 'orig' folder with files in _dest_dir
if [ $comp_unavail -ne 0 ]
then
if [ $# -eq 1 ]
then
echo "Compare software missing."
echo "Install \"ImageMagick\" if you like to procduce \"comp\" files."
echo "Printing diffs to output"
fi
fi
if [ -z "$_dest_dir" ]
then
printf "For comparing, Usage: %s -p -d DirName\nor\n%s -c -d DirName TestName1, TestName2,...\n" $(basename $0) $(basename $0) >&2
fi
if [ "$_dest_dir" = "$_orig_dir" ]
then
printf "Dest-Dir is $_dest_dir, exiting.\n"
return 0
fi
local files_list=
for test_name in $_test_name
do
rm "$_dest_dir"/comp_"$test_name"_[0-9]*.png &> /dev/null
files_list=( `ls "$_dest_dir"/"$test_name"_[0-9]*.png` )
process_compare "${files_list[@]}"
done
if [ "$ncomp" -ne 0 ]
then
echo "Compared $ncomp images."
fi
if [ "$nfail" -ne 0 ]
then
echo "Tests with render regressions: $nfail."
fi
if [ "$nerr" -ne 0 ]
then
echo "$nerr PNG-files were not found"
fi
return 0
}
name_in_args () {
# This function gets curline as first arg
# Then list of args to find if test name is first field of curline
get_test_params "$1"
if [ $? -ne 0 ]
then
return 0
fi
if [ -z "$_test_name" ]
then
return 0
fi
shift
while (( "$#" ));
do
if [ "$_test_name" = "$1" ]
# found test name in list of args
then
return 1
fi
shift
done
# Not found
return 0
}
for_test_in_test_file_do () {
while read curline;
do
name_in_args "$curline" $*
_run_test=$(( $? + $_test_all ))
if [ $_run_test -ne 0 ]
then
$1 "$curline"
if [ $? -ne 0 ]
then
(( _n_exe_err++ ))
fi
fi
done < "$_test_file_name"
}
# Script Entry Point
OUR_LIBPATH="@libdir@/exactness"
_verbose=false
_record=
_play=
_compare=
_init=
_simulation=
_remove_fail=
_orig_dir="orig"
# Init dest_dir - should change on the fly
_dest_dir=
_test_all=1
_base_dir="@datadir@/exactness/recordings"
_test_name=
_test_cmd=
nerr=0
ncomp=0
nfail=0
_n_exe_err=0
# Test that compare is insatlled
which compare &> /dev/null
comp_unavail=$?
while getopts 'ab:d:hprisv?' OPTION
do
case $OPTION in
b) _base_dir="$OPTARG"
;;
d) _dest_dir="$OPTARG"
;;
p) _play=1
_compare=1
_remove_fail=1
;;
r) _record=1
_remove_fail=1
;;
i) _dest_dir="$_orig_dir"
_init=1
_play=1
_remove_fail=1
;;
s) _dest_dir="$_orig_dir"
_simulation=1
;;
h) do_help
exit 0
;;
v) _verbose=true
;;
?) do_help
exit 0
;;
esac
done
shift $(($OPTIND - 1))
_test_file_name="$1"
shift
# Test if user added test-names as arguments
if [ ! -z "$*" ]
then
_test_all=0
fi
# when using -o option, we can loop now on tests names
# given as arguments to this script
DEBUG echo _test_file_name="$_test_file_name"
DEBUG echo _base_dir="$_base_dir"
DEBUG echo _dest_dir="$_dest_dir"