From 0703cc0f5ee76ff607a03ef5b7c452e687cef0e9 Mon Sep 17 00:00:00 2001 From: Daniel Kolesa Date: Tue, 2 Jun 2015 18:12:09 +0100 Subject: [PATCH] eolian: initial lexing of documentation --- src/lib/eolian/eo_lexer.c | 140 ++++++++++++++++++++++++++++++++++--- src/lib/eolian/eo_lexer.h | 25 ++++++- src/lib/eolian/eo_parser.c | 2 +- 3 files changed, 155 insertions(+), 12 deletions(-) diff --git a/src/lib/eolian/eo_lexer.c b/src/lib/eolian/eo_lexer.c index 32be9f90fa..0e9548f3cc 100644 --- a/src/lib/eolian/eo_lexer.c +++ b/src/lib/eolian/eo_lexer.c @@ -49,7 +49,7 @@ static const char * const tokens[] = { "==", "!=", ">=", "<=", "&&", "||", "<<", ">>", - "", "", "", "", "", + "", "", "", "", "", "", KEYWORDS }; @@ -114,7 +114,7 @@ init_hash(void) unsigned int i, u; if (keyword_map) return; keyword_map = eina_hash_string_superfast_new(NULL); - for (i = u = 13; i < (sizeof(tokens) / sizeof(const char*)); ++i) + for (i = u = 14; i < (sizeof(tokens) / sizeof(const char*)); ++i) { eina_hash_add(keyword_map, tokens[i], (void*)(size_t)(i - u + 1)); } @@ -252,6 +252,107 @@ cend: if (tok) tok->value.s = eina_stringshare_add(eina_strbuf_string_get(ls->buff)); } +static void +read_doc(Eo_Lexer *ls, Eo_Token *tok) +{ + Eo_Doc *doc = calloc(1, sizeof(Eo_Doc)); + eina_strbuf_reset(ls->buff); + + skip_ws(ls); + while (is_newline(ls->current)) + next_line_ws(ls); + + for (;;) + { + if (!ls->current) + { + free(doc); + eo_lexer_lex_error(ls, "unfinished documentation", -1); + } + if (is_newline(ls->current)) + { + while (is_newline(ls->current)) + next_line_ws(ls); + break; + } + else + { + if (ls->current == ']') + { + next_char(ls); + if (ls->current != ']') + eina_strbuf_append_char(ls->buff, ']'); + else + { + next_char(ls); + eina_strbuf_trim(ls->buff); + doc->summary = eina_stringshare_add( + eina_strbuf_string_get(ls->buff)); + tok->value.doc = doc; + return; + } + } + eina_strbuf_append_char(ls->buff, ls->current); + next_char(ls); + } + } + + eina_strbuf_trim(ls->buff); + doc->summary = eina_stringshare_add(eina_strbuf_string_get(ls->buff)); + + Eina_Strbuf *rbuf = eina_strbuf_new(); + for (;;) + { + if (!ls->current) + { + eina_stringshare_del(doc->summary); + free(doc); + eina_strbuf_free(rbuf); + eo_lexer_lex_error(ls, "unfinished documentation", -1); + } + + eina_strbuf_reset(ls->buff); + while (ls->current && !is_newline(ls->current)) + { + if (ls->current == ']') + { + next_char(ls); + if (ls->current != ']') + eina_strbuf_append_char(ls->buff, ']'); + else + { + next_char(ls); + eina_strbuf_trim(ls->buff); + eina_strbuf_append(rbuf, eina_strbuf_string_get(ls->buff)); + eina_strbuf_trim(rbuf); + if (eina_strbuf_string_get(rbuf)[0]) + doc->description = eina_stringshare_add( + eina_strbuf_string_get(rbuf)); + eina_strbuf_free(rbuf); + tok->value.doc = doc; + return; + } + } + eina_strbuf_append_char(ls->buff, ls->current); + next_char(ls); + } + eina_strbuf_trim(ls->buff); + eina_strbuf_append(rbuf, eina_strbuf_string_get(ls->buff)); + + if (is_newline(ls->current)) + { + next_line_ws(ls); + /* new paragraph */ + if (is_newline(ls->current)) + eina_strbuf_append(rbuf, "\n\n"); + else + eina_strbuf_append_char(rbuf, ' '); + while (is_newline(ls->current)) + next_line_ws(ls); + } + } +} + static void esc_error(Eo_Lexer *ls, int *c, int n, const char *msg) { @@ -549,6 +650,12 @@ lex(Eo_Lexer *ls, Eo_Token *tok) } continue; } + case '[': + next_char(ls); + if (ls->current != '[') return '['; + next_char(ls); + read_doc(ls, tok); + return TOK_DOC; case '\0': return -1; case '=': @@ -748,6 +855,25 @@ _temps_free(Eo_Lexer_Temps *tmp) if (s) eina_stringshare_del(s); } +static void +_free_tok(Eo_Token *tok) +{ + if (tok->token < START_CUSTOM || tok->token == TOK_NUMBER || + tok->token == TOK_CHAR) + return; + if (tok->token == TOK_DOC) + { + /* free doc */ + eina_stringshare_del(tok->value.doc->summary); + eina_stringshare_del(tok->value.doc->description); + free(tok->value.doc); + tok->value.doc = NULL; + return; + } + eina_stringshare_del(tok->value.s); + tok->value.s = NULL; +} + void eo_lexer_free(Eo_Lexer *ls) { @@ -757,6 +883,7 @@ eo_lexer_free(Eo_Lexer *ls) if (ls->buff ) eina_strbuf_free (ls->buff); if (ls->handle ) eina_file_close (ls->handle); + _free_tok(&ls->t); eo_lexer_context_clear(ls); _temps_free(&ls->tmp); free(ls); @@ -778,12 +905,7 @@ eo_lexer_new(const char *source) int eo_lexer_get(Eo_Lexer *ls) { - if (ls->t.token >= START_CUSTOM && ls->t.token != TOK_NUMBER - && ls->t.token != TOK_CHAR) - { - eina_stringshare_del(ls->t.value.s); - ls->t.value.s = NULL; - } + _free_tok(&ls->t); if (ls->lookahead.token >= 0) { ls->t = ls->lookahead; @@ -850,7 +972,7 @@ eo_lexer_token_to_str(int token, char *buf) const char * eo_lexer_keyword_str_get(int kw) { - return tokens[kw + 12]; + return tokens[kw + 13]; } Eina_Bool diff --git a/src/lib/eolian/eo_lexer.h b/src/lib/eolian/eo_lexer.h index aea3a1c839..f56bc0b61b 100644 --- a/src/lib/eolian/eo_lexer.h +++ b/src/lib/eolian/eo_lexer.h @@ -17,7 +17,7 @@ enum Tokens TOK_EQ = START_CUSTOM, TOK_NQ, TOK_GE, TOK_LE, TOK_AND, TOK_OR, TOK_LSH, TOK_RSH, - TOK_COMMENT, TOK_STRING, TOK_CHAR, TOK_NUMBER, TOK_VALUE + TOK_COMMENT, TOK_DOC, TOK_STRING, TOK_CHAR, TOK_NUMBER, TOK_VALUE }; /* all keywords in eolian, they can still be used as names (they're TOK_VALUE) @@ -81,13 +81,34 @@ enum Numbers NUM_DOUBLE }; +typedef struct _Eo_Doc +{ + Eina_Stringshare *summary; + Eina_Stringshare *description; +} Eo_Doc; + +typedef union +{ + char c; + const char *s; + signed int i; + unsigned int u; + signed long l; + unsigned long ul; + signed long long ll; + unsigned long long ull; + float f; + double d; + Eo_Doc *doc; +} Eo_Token_Union; + /* a token - "token" is the actual token id, "value" is the value of a token * if needed - NULL otherwise - for example the value of a TOK_VALUE, "kw" * is the keyword id if this is a keyword, it's 0 when not a keyword */ typedef struct _Eo_Token { int token, kw; - Eolian_Value_Union value; + Eo_Token_Union value; } Eo_Token; typedef struct _Lexer_Ctx diff --git a/src/lib/eolian/eo_parser.c b/src/lib/eolian/eo_parser.c index 88d1a1e6ab..6a34d2c90c 100644 --- a/src/lib/eolian/eo_parser.c +++ b/src/lib/eolian/eo_parser.c @@ -326,7 +326,7 @@ parse_expr_simple(Eo_Lexer *ls) expr = push_expr(ls); FILL_BASE(expr->base, ls, line, col); expr->type = ls->t.kw + 1; /* map Numbers from lexer to expr type */ - expr->value = ls->t.value; + memcpy(&expr->value, &ls->t.value, sizeof(expr->value)); eo_lexer_get(ls); break; }