enlightenment
/
marrakesh
7
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

make repo db indexer work and server load index and search/list

This commit is contained in:
Carsten Haitzler 2014-09-27 09:58:18 +09:00
parent c331c1f105
commit 44b39a8e3a
3 changed files with 584 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

210
mrk-index.c
View File

@ -29,21 +29,95 @@ read_string(Eet_File *ef, const char *key)
}
static void
key_add(Eina_Hash *hash, const char *key, const char *entry)
key_register(Eina_Hash *hash, const char *key, const char *entry)
{
const char *ss;
Eina_List *entries;
printf("IDX: %p %s -> %s\n", hash, key, entry);
ss = eina_stringshare_add(entry);
entries = eina_hash_find(hash, key);
entries = eina_list_sorted_insert(entries, EINA_COMPARE_CB(strcasecmp), ss);
eina_hash_add(hash, key, entries);
eina_hash_set(hash, key, entries);
}
typedef struct
{
Eina_Hash *master;
const char *entry;
} Inf;
static Eina_Bool
key_enter_cb(const Eina_Hash *hash, const void *key, void *data, void *fdata)
{
Inf *inf = fdata;
key_register(inf->master, key, inf->entry);
return EINA_TRUE;
}
static void
key_str_add(Eina_Hash *hash, const char *string, const char *entry)
key_enter(Eina_Hash *master, Eina_Hash *keys, const char *entry)
{
// XXX: parse "words" from string - add each to the key given
Inf inf = { master, entry };
eina_hash_foreach(keys, key_enter_cb, &inf);
}
static Eina_Hash *
key_new(void)
{
return eina_hash_string_superfast_new(NULL);
}
static void
key_free(Eina_Hash *hash)
{
eina_hash_free(hash);
}
static void
key_add(Eina_Hash *hash, const char *key)
{
if (eina_hash_find(hash, key)) return;
eina_hash_add(hash, key, hash);
}
static void
key_str_add(Eina_Hash *hash, const char *str)
{
char *ts = strdup(str);
char *p, *pe;
if (!ts) return;
for (p = ts; *p; p++)
{
if (!(((*p >= 'a') && (*p <= 'z')) ||
((*p >= 'A') && (*p <= 'Z')) ||
((*p >= '0') && (*p <= '9')))) *p = 0;
else *p = tolower(*p);
pe = p;
}
for (p = ts; p < pe; p++)
{
if (*p)
{
key_add(hash, p);
p += strlen(p);
}
}
free(ts);
}
static Eina_Bool
key_merge_cb(const Eina_Hash *hash, const void *key, void *data, void *fdata)
{
key_add(fdata, key);
return EINA_TRUE;
}
static void
key_merge(Eina_Hash *master, Eina_Hash *keys)
{
eina_hash_foreach(keys, key_merge_cb, master);
}
static int
@ -53,57 +127,101 @@ index_file(const char *file, const char *fname)
char *s;
int i;
char tmp[4096];
Eina_Hash *ky;
Eina_Hash *tk;
printf("--- %s ---\n", file);
ef = eet_open(file, EET_FILE_MODE_READ);
if (!ef) return 0;
ky = key_new();
printf(".. name\n");
s = read_string(ef, "name");
if (s)
{
key_add(db->name_hash, s, fname);
key_add(db->key_hash, s, fname);
free(s);
}
s = read_string(ef, "version");
if (s)
{
key_add(db->key_hash, s, fname);
tk = key_new();
key_add(tk, s);
key_str_add(ky, s);
key_enter(db->name_hash, tk, fname);
key_free(tk);
tk = key_new();
key_str_add(tk, s);
key_merge(ky, tk);
key_free(tk);
free(s);
}
printf(".. tags\n");
for (i = 0; i < 9999; i++)
{
tk = key_new();
snprintf(tmp, sizeof(tmp), "tag/%i", i);
s = read_string(ef, tmp);
if (s)
{
key_add(db->tag_hash, s, fname);
key_add(db->key_hash, s, fname);
key_str_add(tk, s);
free(s);
}
key_enter(db->tag_hash, tk, fname);
key_merge(ky, tk);
key_free(tk);
}
printf(".. categories\n");
key_register(db->category_hash, "@", fname);
for (i = 0; i < 9999; i++)
{
tk = key_new();
snprintf(tmp, sizeof(tmp), "category/%i", i);
s = read_string(ef, tmp);
if (s)
{
key_add(db->category_hash, s, fname);
key_add(db->key_hash, s, fname);
key_str_add(tk, s);
free(s);
}
key_enter(db->category_hash, tk, fname);
key_merge(ky, tk);
key_free(tk);
}
// XXX: weed duplicates from every key
printf(".. brief\n");
s = read_string(ef, "brief");
if (s)
{
tk = key_new();
key_str_add(tk, s);
key_merge(ky, tk);
key_free(tk);
free(s);
}
key_enter(db->key_hash, ky, fname);
key_free(ky);
eet_close(ef);
return 1;
}
static Eina_Bool
index_out_cb(const Eina_Hash *hash, const void *key, void *data, void *fdata)
{
FILE *f = fdata;
const char *s;
fprintf(f, "%s\n", key, data);
EINA_LIST_FREE(data, s)
{
fprintf(f, "\t%s\n", s);
eina_stringshare_del(s);
}
return EINA_TRUE;
}
static int
index_dir(const char *dir)
{
Eina_List *files;
char *s;
char tmp[4096];
FILE *f;
db = calloc(1, sizeof(Mdb));
if (!db) return 0;
@ -111,25 +229,70 @@ index_dir(const char *dir)
db->category_hash = eina_hash_string_superfast_new(NULL);
db->tag_hash = eina_hash_string_superfast_new(NULL);
db->key_hash = eina_hash_string_superfast_new(NULL);
files = ecore_file_ls(dir);
EINA_LIST_FREE(files, s)
{
char *lnk;
if (s[0] == '.') continue;
if (s[0] == '.')
{
free(s);
continue;
}
snprintf(tmp, sizeof(tmp), "%s/%s", dir, s);
lnk = ecore_file_readlink(tmp);
if (!lnk) continue;
if (!lnk)
{
free(s);
continue;
}
index_file(tmp, s);
free(lnk);
free(s);
}
snprintf(tmp, sizeof(tmp), "%s/.mrkdb-name.txt", dir);
f = fopen(tmp, "wb");
if (f)
{
eina_hash_foreach(db->name_hash, index_out_cb, f);
eina_hash_free(db->name_hash);
fclose(f);
}
snprintf(tmp, sizeof(tmp), "%s/.mrkdb-tag.txt", dir);
f = fopen(tmp, "wb");
if (f)
{
eina_hash_foreach(db->tag_hash, index_out_cb, f);
eina_hash_free(db->tag_hash);
fclose(f);
}
snprintf(tmp, sizeof(tmp), "%s/.mrkdb-category.txt", dir);
f = fopen(tmp, "wb");
if (f)
{
eina_hash_foreach(db->category_hash, index_out_cb, f);
eina_hash_free(db->category_hash);
fclose(f);
}
snprintf(tmp, sizeof(tmp), "%s/.mrkdb-key.txt", dir);
f = fopen(tmp, "wb");
if (f)
{
eina_hash_foreach(db->key_hash, index_out_cb, f);
eina_hash_free(db->key_hash);
fclose(f);
}
free(db);
return 1;
}
int
main(int argc, char **argv)
{
int i;
if (argc < 2)
{
printf("usage:\n"
@ -142,8 +305,13 @@ main(int argc, char **argv)
eet_init();
ecore_file_init();
if (argc > 1) repodir = ecore_file_realpath(argv[1]);
index_dir(repodir);
for (i = 1; i < argc; i++)
{
repodir = ecore_file_realpath(argv[i]);
index_dir(repodir);
free(repodir);
repodir = NULL;
}
ecore_file_shutdown();
eet_shutdown();

294
mrk-srv.c
View File

@ -13,6 +13,175 @@ static const char *sane_name_ok = "01234567890-_abcdefghijklmnopqrstuvwxyzABCDEF
static char *repodir = "./repo";
static char *build_sh = "./mrk-srv-build.sh";
typedef struct
{
const char *arch; // also dir
Eina_Hash *name_hash;
Eina_Hash *category_hash;
Eina_Hash *tag_hash;
Eina_Hash *key_hash;
} Db;
static Eina_List *dbs = NULL;
static void
dbs_load_hash(Eina_Hash *hash, FILE *f)
{
char line[16384], *s;
int len;
Eina_List *list = NULL;
const char *key = NULL;
for (;;)
{
s = fgets(line, sizeof(line), f);
if (!s) break;;
len = strlen(s);
if (len < 1) continue;
s[len - 1] = 0;
if ((s[0] == '\t') && (key))
{
list = eina_list_append(list, eina_stringshare_add(s + 1));
}
else
{
if (key) eina_hash_set(hash, key, list);
list = NULL;
if (key) eina_stringshare_del(key);
key = eina_stringshare_add(s);
}
}
if (key) eina_hash_set(hash, key, list);
list = NULL;
if (key) eina_stringshare_del(key);
key = NULL;
}
static void
dbs_load(void)
{
Eina_List *files;
char *s;
char tmp[4096];
files = ecore_file_ls(repodir);
EINA_LIST_FREE(files, s)
{
Db *db = NULL;
FILE *f;
if (s[0] == '.')
{
free(s);
continue;
}
snprintf(tmp, sizeof(tmp), "%s/%s/.mrkdb-name.txt", repodir, s);
f = fopen(tmp, "rb");
if (f)
{
db = calloc(1, sizeof(Db));
if (db)
{
db->arch = strdup(s);
if (db->arch)
{
db->name_hash = eina_hash_string_superfast_new(NULL);
db->category_hash = eina_hash_string_superfast_new(NULL);
db->tag_hash = eina_hash_string_superfast_new(NULL);
db->key_hash = eina_hash_string_superfast_new(NULL);
dbs = eina_list_append(dbs, db);
dbs_load_hash(db->name_hash, f);
}
else
{
free(db);
db = NULL;
}
}
fclose(f);
}
if (db)
{
snprintf(tmp, sizeof(tmp), "%s/%s/.mrkdb-tag.txt", repodir, s);
f = fopen(tmp, "rb");
if (f)
{
dbs_load_hash(db->tag_hash, f);
fclose(f);
}
snprintf(tmp, sizeof(tmp), "%s/%s/.mrkdb-category.txt", repodir, s);
f = fopen(tmp, "rb");
if (f)
{
dbs_load_hash(db->category_hash, f);
fclose(f);
}
snprintf(tmp, sizeof(tmp), "%s/%s/.mrkdb-key.txt", repodir, s);
f = fopen(tmp, "rb");
if (f)
{
dbs_load_hash(db->key_hash, f);
fclose(f);
}
}
db = NULL;
}
}
static Eina_Bool
dbs_clear_hash_cb(const Eina_Hash *hash, const void *key, void *data, void *fdata)
{
const char *s;
EINA_LIST_FREE(data, s)
{
eina_stringshare_del(s);
}
return EINA_TRUE;
}
static void
dbs_clear_hash(Eina_Hash *hash)
{
if (!hash) return;
eina_hash_foreach(hash, dbs_clear_hash_cb, NULL);
eina_hash_free(hash);
}
static void
dbs_clear(void)
{
Db *db;
EINA_LIST_FREE(dbs, db)
{
dbs_clear_hash(db->name_hash);
dbs_clear_hash(db->category_hash);
dbs_clear_hash(db->tag_hash);
dbs_clear_hash(db->key_hash);
free(db);
}
}
static Db *
dbs_arch_find(const char *arch)
{
Eina_List *l;
Db *db;
if (!arch) return NULL;
EINA_LIST_FOREACH(dbs, l, db)
{
if (!strcmp(db->arch, arch))
{
if (dbs != l) dbs = eina_list_promote_list(dbs, l);
return db;
}
}
return NULL;
}
static int
sane_forbidden_path(const char *file, const char **forbidden)
{
@ -263,6 +432,64 @@ version_ok(const char *arch)
return 1;
}
static Eina_List *
search_intersect(Eina_List *list_in, Eina_List *list2)
{
Eina_List *list = NULL, *ll;
const char *s, *ss;
int ok;
EINA_LIST_FREE(list_in, s)
{
ok = 0;
EINA_LIST_FOREACH(list2, ll, ss)
{
if (ss == s)
{
ok = 1;
break;
}
}
if (ok) list = eina_list_append(list, s);
else eina_stringshare_del(s);
}
return list;
}
static Eina_List *
search_build(Eina_Hash *hash, const char *search)
{
Eina_List *list = NULL, *tl, *l;
const char *s;
char *ts = strdup(search);
char *p, *end;
int i;
if (!ts) return NULL;
for (p = ts; *p; p++)
{
if (isspace(*p)) *p = 0;
end = p;
}
for (p = ts; p < end; p++)
{
if (!*p) continue;
tl = eina_hash_find(hash, p);
if (p == ts)
{
EINA_LIST_FOREACH(tl, l, s)
list = eina_list_append(list, eina_stringshare_add(s));
}
else
{
list = search_intersect(list, tl);
}
p += strlen(p);
}
free(ts);
return list;
}
static Eina_Bool
_ipc_cb_add(void *data, int type, void *event)
{
@ -321,9 +548,73 @@ _ipc_cb_dat(void *data, int type, void *event)
break;
case M_QRY_LIST:
// e->data == category (none if all)
ecore_ipc_client_send(c->client, 10, M_ANS_START, 0, 0, 0, NULL, 0);
if ((e->size > 0) && (e->size < 1000))
{
Eina_List *list = NULL, *l;
const char *s;
Db *db = dbs_arch_find(c->arch);
if (db)
{
char *str = malloc(e->size + 1);
if (str)
{
memcpy(str, e->data, e->size);
str[e->size] = 0;
list = eina_hash_find(db->category_hash, str);
free(str);
EINA_LIST_FOREACH(list, l, s)
{
ecore_ipc_client_send(c->client, 10, M_ANS_DATA, 0, 0, 0, s, strlen(s));
}
}
}
}
else
{
Eina_List *list = NULL, *l;
const char *s;
Db *db = dbs_arch_find(c->arch);
if (db)
{
list = eina_hash_find(db->category_hash, "@");
EINA_LIST_FOREACH(list, l, s)
{
ecore_ipc_client_send(c->client, 10, M_ANS_DATA, 0, 0, 0, s, strlen(s));
}
}
}
ecore_ipc_client_send(c->client, 10, M_ANS_END, 0, 0, 0, NULL, 0);
break;
case M_QRY_SEARCH:
// e->data == query string
ecore_ipc_client_send(c->client, 10, M_ANS_START, 0, 0, 0, NULL, 0);
if ((e->size > 0) && (e->size < 1000))
{
Eina_List *list = NULL;
const char *s;
Db *db = dbs_arch_find(c->arch);
if (db)
{
char *str = malloc(e->size + 1);
if (str)
{
memcpy(str, e->data, e->size);
str[e->size] = 0;
list = search_build(db->key_hash, str);
free(str);
EINA_LIST_FREE(list, s)
{
ecore_ipc_client_send(c->client, 10, M_ANS_DATA, 0, 0, 0, s, strlen(s));
eina_stringshare_del(s);
}
}
}
}
ecore_ipc_client_send(c->client, 10, M_ANS_END, 0, 0, 0, NULL, 0);
break;
case M_QRY_GET:
if ((e->size > 0) && (e->size < 1000))
@ -380,7 +671,6 @@ _ipc_cb_dat(void *data, int type, void *event)
{
memcpy(arch, e->data, e->size);
arch[e->size] = 0;
printf("set arch %s\n", arch);
if (arch_ok(arch))
{
if (c->arch) free(c->arch);
@ -443,6 +733,8 @@ main(int argc, char **argv)
if (argc > 1) repodir = ecore_file_realpath(argv[1]);
if (argc > 2) build_sh = ecore_file_realpath(argv[2]);
dbs_load();
if (ipc_init())
{
ecore_exe_run_priority_set(10);

104
mrk.c
View File

@ -1347,6 +1347,79 @@ package_downsrc(const char *name)
return 1;
}
static int qry_op = 0;
static char *qry_data;
static Eina_Bool
_ipc4_cb_add(void *data, int type, void *event)
{
Ecore_Ipc_Event_Server_Add *e = event;
char tmp[4096];
snprintf(tmp, sizeof(tmp), "%s-%s", os, arch);
ecore_ipc_server_send(ipc, 10, M_ID_ARCH, 0, 0, 0,
tmp, strlen(tmp));
snprintf(tmp, sizeof(tmp), "%i.%i.%i",
eina_version->major, eina_version->minor, eina_version->micro);
ecore_ipc_server_send(ipc, 10, M_ID_VERSION, 0, 0, 0,
tmp, strlen(tmp));
ecore_ipc_server_send(ipc, 10, qry_op, 0, 0, 0,
qry_data, qry_data ? strlen(qry_data) : 0);
return EINA_TRUE;
}
static Eina_Bool
_ipc4_cb_del(void *data, int type, void *event)
{
Ecore_Ipc_Event_Server_Del *e = event;
err("disconnect...");
ecore_main_loop_quit();
return EINA_TRUE;
}
static Eina_Bool
_ipc4_cb_dat(void *data, int type, void *event)
{
Ecore_Ipc_Event_Server_Data *e = event;
if (e->major == 10)
{
switch (e->minor)
{
case M_ANS_START:
break;
case M_ANS_DATA:
if ((e->data) && (e->size > 0) && (e->size <= 10000))
{
fwrite(e->data, e->size, 1, stdout);
fwrite("\n", 1, 1, stdout);
}
break;
case M_ANS_END:
if (qry_data) free(qry_data);
qry_data = NULL;
ecore_main_loop_quit();
break;
default:
break;
}
}
return EINA_TRUE;
}
static int
package_qry(int op, const char *opdat)
{
ipc = ecore_ipc_server_connect(ECORE_IPC_REMOTE_SYSTEM, server, server_port, NULL);
if (!ipc) return 0;
ecore_event_handler_add(ECORE_IPC_EVENT_SERVER_ADD, _ipc4_cb_add, NULL);
ecore_event_handler_add(ECORE_IPC_EVENT_SERVER_DEL, _ipc4_cb_del, NULL);
ecore_event_handler_add(ECORE_IPC_EVENT_SERVER_DATA, _ipc4_cb_dat, NULL);
qry_op = op;
if (opdat) qry_data = strdup(opdat);
ecore_main_loop_begin();
return 1;
}
int
main(int argc, char **argv)
{
@ -1369,8 +1442,8 @@ main(int argc, char **argv)
" rel [FILE]\n"
" down PKGNAME\n"
" get PKGNAME\n"
// " list [CATEGORY]\n"
// " search KEY1 [KEY2] [KEY3] [...]\n"
" list [CATEGORY]\n"
" search KEY1 [KEY2] [KEY3] [...]\n"
// " new\n"
// " getsrc\n"
" check\n"
@ -1456,9 +1529,36 @@ main(int argc, char **argv)
}
else if (!strcmp(argv[1], "list"))
{
if (argc < 3) package_qry(M_QRY_LIST, NULL);
else package_qry(M_QRY_LIST, argv[2]);
}
else if (!strcmp(argv[1], "search"))
{
if (argc < 3) err("need search strings");
else
{
int i;
int len = 0;
char *ts;
for (i = 2; i < argc; i++)
{
len += strlen(argv[i]);
len++;
}
ts = malloc(len);
if (ts)
{
ts[0] = 0;
for (i = 2; i < argc; i++)
{
strcat(ts, argv[i]);
if (i < (argc - 1)) strcat(ts, " ");
}
package_qry(M_QRY_SEARCH, ts);
free(ts);
}
}
}
else if (!strcmp(argv[1], "new"))
{