summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDave Andreoli <dave@gurumeditation.it>2018-01-02 13:57:22 +0100
committerDave Andreoli <dave@gurumeditation.it>2018-01-02 13:58:08 +0100
commit3388077bc35b9d0be902051aee3d9977562dda6c (patch)
tree1d9189be699d4f4ae054c2492c5291e33507d507
parent9bedda14b3a3edb972bb231dd1e727c86355d6da (diff)
Pyolian: implemented doc tokenizer
-rw-r--r--src/scripts/pyolian/eolian.py69
-rw-r--r--src/scripts/pyolian/eolian_lib.py2
-rwxr-xr-xsrc/scripts/pyolian/generator.py1
3 files changed, 68 insertions, 4 deletions
diff --git a/src/scripts/pyolian/eolian.py b/src/scripts/pyolian/eolian.py
index a5653a5120..75a98ff0fe 100644
--- a/src/scripts/pyolian/eolian.py
+++ b/src/scripts/pyolian/eolian.py
@@ -24,13 +24,13 @@ a way that this folder will be available on PYTHON_PATH, fe:
24 24
25""" 25"""
26from enum import IntEnum 26from enum import IntEnum
27from ctypes import cast, byref, c_char_p, c_void_p 27from ctypes import cast, byref, c_char_p, c_void_p, c_int
28import ctypes
28 29
29try: 30try:
30 from .eolian_lib import lib 31 from .eolian_lib import lib
31except ImportError: 32except ImportError:
32 from eolian_lib import lib 33 from eolian_lib import lib
33
34 34
35 35
36### Eolian Enums ############################################################ 36### Eolian Enums ############################################################
@@ -1235,11 +1235,16 @@ class Declaration(EolianBaseObject):
1235 return Variable(c_var) if c_var else None 1235 return Variable(c_var) if c_var else None
1236 1236
1237 1237
1238class _Eolian_Doc_Token_Struct(ctypes.Structure):
1239 _fields_ = [("type", c_int),
1240 ("text", c_char_p),
1241 ("text_end", c_char_p)]
1242
1238class Documentation(EolianBaseObject): 1243class Documentation(EolianBaseObject):
1239 # def __repr__(self): 1244 # def __repr__(self):
1240 # return "<eolian.Documentation '{0.name}'>".format(self) 1245 # return "<eolian.Documentation '{0.name}'>".format(self)
1241 1246
1242 # this is too much for py, just use string.split('\n\n') 1247 # this is too much for py, just use string.split('\n\n') instead
1243 # def string_split(self, string): 1248 # def string_split(self, string):
1244 # c_list = lib.eolian_documentation_string_split 1249 # c_list = lib.eolian_documentation_string_split
1245 1250
@@ -1255,6 +1260,64 @@ class Documentation(EolianBaseObject):
1255 def since(self): 1260 def since(self):
1256 return _str_to_py(lib.eolian_documentation_since_get(self._obj)) 1261 return _str_to_py(lib.eolian_documentation_since_get(self._obj))
1257 1262
1263 @property
1264 def summary_tokens(self):
1265 """ return a list of paragraphs, each one is a list of tokens """
1266 return self._tokenize(self.summary)
1267
1268 @property
1269 def description_tokens(self):
1270 """ return a list of paragraphs, each one is a list of tokens """
1271 return self._tokenize(self.description)
1272
1273 @classmethod
1274 def _tokenize(cls, full_text):
1275 paragraphs = []
1276 if not full_text:
1277 return paragraphs
1278
1279 tok = _Eolian_Doc_Token_Struct()
1280 for paragraph in full_text.split('\n\n'):
1281 tokens = []
1282 c_paragraph = _str_to_bytes(paragraph) # keep c_paragraph alive !
1283
1284 lib.eolian_doc_token_init(byref(tok))
1285 next_chunk = lib.eolian_documentation_tokenize(c_paragraph, byref(tok))
1286 while next_chunk:
1287 typ = lib.eolian_doc_token_type_get(byref(tok))
1288 txt = lib.eolian_doc_token_text_get(byref(tok))
1289 # ref = # TODO ... Stupido parametro '*unit' :(
1290 tokens.append(Documentation_Token(typ, txt))
1291 lib.free(c_void_p(txt))
1292 next_chunk = lib.eolian_documentation_tokenize(c_char_p(next_chunk), byref(tok))
1293 paragraphs.append(tokens)
1294
1295 return paragraphs
1296
1297
1298class Documentation_Token(object):
1299 def __init__(self, c_token_type, c_text):
1300 self._type = Eolian_Doc_Token_Type(c_token_type)
1301 self._text = _str_to_py(c_text)
1302 self._ref = None # TODO
1303
1304 def __repr__(self):
1305 t = self.text if len(self.text) < 40 else self.text[:40] + '...'
1306 return "<eolian.Doc_Token ({}), text='{}', len={}>".format(
1307 self.type.name, t, len(self.text))
1308
1309 @property
1310 def type(self):
1311 return self._type
1312
1313 @property
1314 def text(self):
1315 return self._text
1316
1317 @property
1318 def ref(self):
1319 return self._ref
1320
1258 1321
1259### internal string encode/decode ########################################### 1322### internal string encode/decode ###########################################
1260 1323
diff --git a/src/scripts/pyolian/eolian_lib.py b/src/scripts/pyolian/eolian_lib.py
index a330853dbd..66b1556b5b 100644
--- a/src/scripts/pyolian/eolian_lib.py
+++ b/src/scripts/pyolian/eolian_lib.py
@@ -737,7 +737,7 @@ lib.eolian_documentation_since_get.restype = c_char_p
737 737
738# # EAPI const char *eolian_documentation_tokenize(const char *doc, Eolian_Doc_Token *ret); 738# # EAPI const char *eolian_documentation_tokenize(const char *doc, Eolian_Doc_Token *ret);
739lib.eolian_documentation_tokenize.argtypes = [c_char_p, c_void_p] 739lib.eolian_documentation_tokenize.argtypes = [c_char_p, c_void_p]
740lib.eolian_documentation_tokenize.restype = c_char_p 740lib.eolian_documentation_tokenize.restype = c_void_p # this need to be passed back as char*
741 741
742# EAPI void eolian_doc_token_init(Eolian_Doc_Token *tok); 742# EAPI void eolian_doc_token_init(Eolian_Doc_Token *tok);
743lib.eolian_doc_token_init.argtypes = [c_void_p,] 743lib.eolian_doc_token_init.argtypes = [c_void_p,]
diff --git a/src/scripts/pyolian/generator.py b/src/scripts/pyolian/generator.py
index 84290b6bef..2640bfaf40 100755
--- a/src/scripts/pyolian/generator.py
+++ b/src/scripts/pyolian/generator.py
@@ -127,6 +127,7 @@ class Template(pyratemp.Template):
127 'Variable': eolian.Variable, 127 'Variable': eolian.Variable,
128 'Declaration': eolian.Declaration, 128 'Declaration': eolian.Declaration,
129 'Documentation': eolian.Documentation, 129 'Documentation': eolian.Documentation,
130 'Documentation_Token': eolian.Documentation_Token,
130 # Eolian Enums 131 # Eolian Enums
131 'Eolian_Function_Type': eolian.Eolian_Function_Type, 132 'Eolian_Function_Type': eolian.Eolian_Function_Type,
132 'Eolian_Parameter_Dir': eolian.Eolian_Parameter_Dir, 133 'Eolian_Parameter_Dir': eolian.Eolian_Parameter_Dir,