summaryrefslogtreecommitdiff
path: root/src/scripts/pyolian/pyratemp.py
diff options
context:
space:
mode:
authorDave Andreoli <dave@gurumeditation.it>2017-12-28 16:10:04 +0100
committerDave Andreoli <dave@gurumeditation.it>2017-12-28 16:10:04 +0100
commit15c7bbf5fab8d9d7478b906323fde2c6c147a6d7 (patch)
tree173c60b1c379a04ec666b2ee677d32df3c2e5fff /src/scripts/pyolian/pyratemp.py
parent455f2414635237ed8cebbd1d8ab324d8ec7c167f (diff)
Pyolian template-based generator.
This is a really powerfull tool that can be used to generate anything eolian releted just providing a template file. You can then render the template with the wanted scope (class, namespace, enum, etc) For example give a try at this (from the src/srcipts/pyolian folder): ./generator.py test_gen_class.template --cls Efl.Loop.Timer or ./generator.py -h for the full help Next step: maybe generate the new efl API doc using this tool? @andy I think this will make your life much easier :)
Diffstat (limited to 'src/scripts/pyolian/pyratemp.py')
-rw-r--r--src/scripts/pyolian/pyratemp.py1258
1 files changed, 1258 insertions, 0 deletions
diff --git a/src/scripts/pyolian/pyratemp.py b/src/scripts/pyolian/pyratemp.py
new file mode 100644
index 0000000000..c28e260b4b
--- /dev/null
+++ b/src/scripts/pyolian/pyratemp.py
@@ -0,0 +1,1258 @@
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3"""
4Small, simple and powerful template-engine for Python.
5
6A template-engine for Python, which is very simple, easy to use, small,
7fast, powerful, modular, extensible, well documented and pythonic.
8
9See documentation for a list of features, template-syntax etc.
10
11:Version: 0.3.2
12:Requires: Python >=2.6 / 3.x
13
14:Usage:
15 see class ``Template`` and examples below.
16
17:Example:
18
19 Note that the examples are in Python 2; they also work in
20 Python 3 if you replace u"..." by "...", unicode() by str()
21 and partly "..." by b"...".
22
23 quickstart::
24 >>> t = Template("hello @!name!@")
25 >>> print(t(name="marvin"))
26 hello marvin
27
28 quickstart with a template-file::
29 # >>> t = Template(filename="mytemplate.tmpl")
30 # >>> print(t(name="marvin"))
31 # hello marvin
32
33 generic usage::
34 >>> t = Template(u"output is in Unicode \\xe4\\xf6\\xfc\\u20ac")
35 >>> t #doctest: +ELLIPSIS
36 <...Template instance at 0x...>
37 >>> t()
38 u'output is in Unicode \\xe4\\xf6\\xfc\\u20ac'
39 >>> unicode(t)
40 u'output is in Unicode \\xe4\\xf6\\xfc\\u20ac'
41
42 with data::
43 >>> t = Template("hello @!name!@", data={"name":"world"})
44 >>> t()
45 u'hello world'
46 >>> t(name="worlds")
47 u'hello worlds'
48
49 # >>> t(note="data must be Unicode or ASCII", name=u"\\xe4")
50 # u'hello \\xe4'
51
52 escaping::
53 >>> t = Template("hello escaped: @!name!@, unescaped: $!name!$")
54 >>> t(name='''<>&'"''')
55 u'hello escaped: &lt;&gt;&amp;&#39;&quot;, unescaped: <>&\\'"'
56
57 result-encoding::
58 # encode the unicode-object to your encoding with encode()
59 >>> t = Template(u"hello \\xe4\\xf6\\xfc\\u20ac")
60 >>> result = t()
61 >>> result
62 u'hello \\xe4\\xf6\\xfc\\u20ac'
63 >>> result.encode("utf-8")
64 'hello \\xc3\\xa4\\xc3\\xb6\\xc3\\xbc\\xe2\\x82\\xac'
65 >>> result.encode("ascii")
66 Traceback (most recent call last):
67 ...
68 UnicodeEncodeError: 'ascii' codec can't encode characters in position 6-9: ordinal not in range(128)
69 >>> result.encode("ascii", 'xmlcharrefreplace')
70 'hello &#228;&#246;&#252;&#8364;'
71
72 Python-expressions::
73 >>> Template('formatted: @! "%8.5f" % value !@')(value=3.141592653)
74 u'formatted: 3.14159'
75 >>> Template("hello --@!name.upper().center(20)!@--")(name="world")
76 u'hello -- WORLD --'
77 >>> Template("calculate @!var*5+7!@")(var=7)
78 u'calculate 42'
79
80 blocks (if/for/macros/...)::
81 >>> t = Template("<!--(if foo == 1)-->bar<!--(elif foo == 2)-->baz<!--(else)-->unknown(@!foo!@)<!--(end)-->")
82 >>> t(foo=2)
83 u'baz'
84 >>> t(foo=5)
85 u'unknown(5)'
86
87 >>> t = Template("<!--(for i in mylist)-->@!i!@ <!--(else)-->(empty)<!--(end)-->")
88 >>> t(mylist=[])
89 u'(empty)'
90 >>> t(mylist=[1,2,3])
91 u'1 2 3 '
92
93 >>> t = Template("<!--(for i,elem in enumerate(mylist))--> - @!i!@: @!elem!@<!--(end)-->")
94 >>> t(mylist=["a","b","c"])
95 u' - 0: a - 1: b - 2: c'
96
97 >>> t = Template('<!--(macro greetings)-->hello <strong>@!name!@</strong><!--(end)--> @!greetings(name=user)!@')
98 >>> t(user="monty")
99 u' hello <strong>monty</strong>'
100
101 exists::
102 >>> t = Template('<!--(if exists("foo"))-->YES<!--(else)-->NO<!--(end)-->')
103 >>> t()
104 u'NO'
105 >>> t(foo=1)
106 u'YES'
107 >>> t(foo=None) # note this difference to 'default()'
108 u'YES'
109
110 default-values::
111 # non-existing variables raise an error
112 >>> Template('hi @!optional!@')()
113 Traceback (most recent call last):
114 ...
115 TemplateRenderError: Cannot eval expression 'optional'. (NameError: name 'optional' is not defined)
116
117 >>> t = Template('hi @!default("optional","anyone")!@')
118 >>> t()
119 u'hi anyone'
120 >>> t(optional=None)
121 u'hi anyone'
122 >>> t(optional="there")
123 u'hi there'
124
125 # the 1st parameter can be any eval-expression
126 >>> t = Template('@!default("5*var1+var2","missing variable")!@')
127 >>> t(var1=10)
128 u'missing variable'
129 >>> t(var1=10, var2=2)
130 u'52'
131
132 # also in blocks
133 >>> t = Template('<!--(if default("opt1+opt2",0) > 0)-->yes<!--(else)-->no<!--(end)-->')
134 >>> t()
135 u'no'
136 >>> t(opt1=23, opt2=42)
137 u'yes'
138
139 >>> t = Template('<!--(for i in default("optional_list",[]))-->@!i!@<!--(end)-->')
140 >>> t()
141 u''
142 >>> t(optional_list=[1,2,3])
143 u'123'
144
145
146 # but make sure to put the expression in quotation marks, otherwise:
147 >>> Template('@!default(optional,"fallback")!@')()
148 Traceback (most recent call last):
149 ...
150 TemplateRenderError: Cannot eval expression 'default(optional,"fallback")'. (NameError: name 'optional' is not defined)
151
152 setvar::
153 >>> t = Template('$!setvar("i", "i+1")!$@!i!@')
154 >>> t(i=6)
155 u'7'
156
157 >>> t = Template('''<!--(if isinstance(s, (list,tuple)))-->$!setvar("s", '"\\\\\\\\n".join(s)')!$<!--(end)-->@!s!@''')
158 >>> t(isinstance=isinstance, s="123")
159 u'123'
160 >>> t(isinstance=isinstance, s=["123", "456"])
161 u'123\\n456'
162
163:Author: Roland Koebler (rk at simple-is-better dot org)
164:Copyright: Roland Koebler
165:License: MIT/X11-like, see __license__
166
167:RCS: $Id: pyratemp.py,v 1.22 2013/09/17 07:44:13 rk Exp $
168"""
169from __future__ import unicode_literals
170
171__version__ = "0.3.2"
172__author__ = "Roland Koebler <rk at simple-is-better dot org>"
173__license__ = """Copyright (c) Roland Koebler, 2007-2013
174
175Permission is hereby granted, free of charge, to any person obtaining a copy
176of this software and associated documentation files (the "Software"), to deal
177in the Software without restriction, including without limitation the rights
178to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
179copies of the Software, and to permit persons to whom the Software is
180furnished to do so, subject to the following conditions:
181
182The above copyright notice and this permission notice shall be included in
183all copies or substantial portions of the Software.
184
185THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
186IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
187FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
188AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
189LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
190FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
191IN THE SOFTWARE."""
192
193#=========================================
194
195import os, re, sys, types
196if sys.version_info[0] >= 3:
197 import builtins
198 unicode = str
199 long = int
200else:
201 import __builtin__ as builtins
202 from codecs import open
203
204#=========================================
205# some useful functions
206
207#----------------------
208# string-position: i <-> row,col
209
210def srow(string, i):
211 """Get line numer of ``string[i]`` in `string`.
212
213 :Returns: row, starting at 1
214 :Note: This works for text-strings with ``\\n`` or ``\\r\\n``.
215 """
216 return string.count('\n', 0, max(0, i)) + 1
217
218def scol(string, i):
219 """Get column number of ``string[i]`` in `string`.
220
221 :Returns: column, starting at 1 (but may be <1 if i<0)
222 :Note: This works for text-strings with ``\\n`` or ``\\r\\n``.
223 """
224 return i - string.rfind('\n', 0, max(0, i))
225
226def sindex(string, row, col):
227 """Get index of the character at `row`/`col` in `string`.
228
229 :Parameters:
230 - `row`: row number, starting at 1.
231 - `col`: column number, starting at 1.
232 :Returns: ``i``, starting at 0 (but may be <1 if row/col<0)
233 :Note: This works for text-strings with '\\n' or '\\r\\n'.
234 """
235 n = 0
236 for _ in range(row-1):
237 n = string.find('\n', n) + 1
238 return n+col-1
239
240#----------------------
241
242def dictkeyclean(d):
243 """Convert all keys of the dict `d` to strings.
244 """
245 new_d = {}
246 for k, v in d.items():
247 new_d[str(k)] = v
248 return new_d
249
250#----------------------
251
252def dummy(*_, **__):
253 """Dummy function, doing nothing.
254 """
255 pass
256
257def dummy_raise(exception, value):
258 """Create an exception-raising dummy function.
259
260 :Returns: dummy function, raising ``exception(value)``
261 """
262 def mydummy(*_, **__):
263 raise exception(value)
264 return mydummy
265
266#=========================================
267# escaping
268
269(NONE, HTML, LATEX, MAIL_HEADER) = range(0, 4)
270ESCAPE_SUPPORTED = {"NONE":None, "HTML":HTML, "LATEX":LATEX, "MAIL_HEADER":MAIL_HEADER}
271
272def escape(s, format=HTML):
273 """Replace special characters by their escape sequence.
274
275 :Parameters:
276 - `s`: unicode-string to escape
277 - `format`:
278
279 - `NONE`: nothing is replaced
280 - `HTML`: replace &<>'" by &...;
281 - `LATEX`: replace \#$%&_{}~^
282 - `MAIL_HEADER`: escape non-ASCII mail-header-contents
283 :Returns:
284 the escaped string in unicode
285 :Exceptions:
286 - `ValueError`: if `format` is invalid.
287
288 :Uses:
289 MAIL_HEADER uses module email
290 """
291 #Note: If you have to make sure that every character gets replaced
292 # only once (and if you cannot achieve this with the following code),
293 # use something like "".join([replacedict.get(c,c) for c in s])
294 # which is about 2-3 times slower (but maybe needs less memory).
295 #Note: This is one of the most time-consuming parts of the template.
296 if format is None or format == NONE:
297 pass
298 elif format == HTML:
299 s = s.replace("&", "&amp;") # must be done first!
300 s = s.replace("<", "&lt;")
301 s = s.replace(">", "&gt;")
302 s = s.replace('"', "&quot;")
303 s = s.replace("'", "&#39;")
304 elif format == LATEX:
305 s = s.replace("\\", "\\x") #must be done first!
306 s = s.replace("#", "\\#")
307 s = s.replace("$", "\\$")
308 s = s.replace("%", "\\%")
309 s = s.replace("&", "\\&")
310 s = s.replace("_", "\\_")
311 s = s.replace("{", "\\{")
312 s = s.replace("}", "\\}")
313 s = s.replace("\\x","\\textbackslash{}")
314 s = s.replace("~", "\\textasciitilde{}")
315 s = s.replace("^", "\\textasciicircum{}")
316 elif format == MAIL_HEADER:
317 import email.header
318 try:
319 s.encode("ascii")
320 return s
321 except UnicodeEncodeError:
322 return email.header.make_header([(s, "utf-8")]).encode()
323 else:
324 raise ValueError('Invalid format (only None, HTML, LATEX and MAIL_HEADER are supported).')
325 return s
326
327#=========================================
328
329#-----------------------------------------
330# Exceptions
331
332class TemplateException(Exception):
333 """Base class for template-exceptions."""
334 pass
335
336class TemplateParseError(TemplateException):
337 """Template parsing failed."""
338 def __init__(self, err, errpos):
339 """
340 :Parameters:
341 - `err`: error-message or exception to wrap
342 - `errpos`: ``(filename,row,col)`` where the error occured.
343 """
344 self.err = err
345 self.filename, self.row, self.col = errpos
346 TemplateException.__init__(self)
347 def __str__(self):
348 if not self.filename:
349 return "line %d, col %d: %s" % (self.row, self.col, str(self.err))
350 else:
351 return "file %s, line %d, col %d: %s" % (self.filename, self.row, self.col, str(self.err))
352
353class TemplateSyntaxError(TemplateParseError, SyntaxError):
354 """Template syntax-error."""
355 pass
356
357class TemplateIncludeError(TemplateParseError):
358 """Template 'include' failed."""
359 pass
360
361class TemplateRenderError(TemplateException):
362 """Template rendering failed."""
363 pass
364
365#-----------------------------------------
366# Loader
367
368class LoaderString:
369 """Load template from a string/unicode.
370
371 Note that 'include' is not possible in such templates.
372 """
373 def __init__(self, encoding='utf-8'):
374 self.encoding = encoding
375
376 def load(self, s):
377 """Return template-string as unicode.
378 """
379 if isinstance(s, unicode):
380 u = s
381 else:
382 u = s.decode(self.encoding)
383 return u
384
385class LoaderFile:
386 """Load template from a file.
387
388 When loading a template from a file, it's possible to including other
389 templates (by using 'include' in the template). But for simplicity
390 and security, all included templates have to be in the same directory!
391 (see ``allowed_path``)
392 """
393 def __init__(self, allowed_path=None, encoding='utf-8'):
394 """Init the loader.
395
396 :Parameters:
397 - `allowed_path`: path of the template-files
398 - `encoding`: encoding of the template-files
399 :Exceptions:
400 - `ValueError`: if `allowed_path` is not a directory
401 """
402 if allowed_path and not os.path.isdir(allowed_path):
403 raise ValueError("'allowed_path' has to be a directory.")
404 self.path = allowed_path
405 self.encoding = encoding
406
407 def load(self, filename):
408 """Load a template from a file.
409
410 Check if filename is allowed and return its contens in unicode.
411
412 :Parameters:
413 - `filename`: filename of the template without path
414 :Returns:
415 the contents of the template-file in unicode
416 :Exceptions:
417 - `ValueError`: if `filename` contains a path
418 """
419 if filename != os.path.basename(filename):
420 raise ValueError("No path allowed in filename. (%s)" %(filename))
421 filename = os.path.join(self.path, filename)
422
423 f = open(filename, 'r', encoding=self.encoding)
424 u = f.read()
425 f.close()
426
427 return u
428
429#-----------------------------------------
430# Parser
431
432class Parser(object):
433 """Parse a template into a parse-tree.
434
435 Includes a syntax-check, an optional expression-check and verbose
436 error-messages.
437
438 See documentation for a description of the parse-tree.
439 """
440 # template-syntax (original)
441 # _comment_start = "#!"
442 # _comment_end = "!#"
443 # _sub_start = "$!"
444 # _sub_end = "!$"
445 # _subesc_start = "@!"
446 # _subesc_end = "!@"
447 # _block_start = "<!--("
448 # _block_end = ")-->"
449
450 # template-syntax (eolian)
451 _comment_start = "#!"
452 _comment_end = "!#"
453 _sub_start = "${" # "$!"
454 _sub_end = "}$" # "!$"
455 _subesc_start = "${!" # "@!"
456 _subesc_end = "!}$" # "!@"
457 _block_start = "<!--("
458 _block_end = ")-->"
459
460 # template-syntax (Jinja2 style)
461 # _comment_start = "<!--" # "#!"
462 # _comment_end = "-->" # "!#"
463 # _sub_start = "{{" # "$!"
464 # _sub_end = "}}" # "!$"
465 # _subesc_start = "{!" # "@!"
466 # _subesc_end = "!}" # "!@"
467 # _block_start = "{% " # "<!--("
468 # _block_end = " %}" # ")-->"
469
470 # build regexps
471 # comment
472 # single-line, until end-tag or end-of-line.
473 _strComment = r"""%s(?P<content>.*?)(?P<end>%s|\n|$)""" \
474 % (re.escape(_comment_start), re.escape(_comment_end))
475 _reComment = re.compile(_strComment, re.M)
476
477 # escaped or unescaped substitution
478 # single-line ("|$" is needed to be able to generate good error-messges)
479 _strSubstitution = r"""
480 (
481 %s\s*(?P<sub>.*?)\s*(?P<end>%s|$) #substitution
482 |
483 %s\s*(?P<escsub>.*?)\s*(?P<escend>%s|$) #escaped substitution
484 )
485 """ % (re.escape(_sub_start), re.escape(_sub_end),
486 re.escape(_subesc_start), re.escape(_subesc_end))
487 _reSubstitution = re.compile(_strSubstitution, re.X|re.M)
488
489 # block
490 # - single-line, no nesting.
491 # or
492 # - multi-line, nested by whitespace indentation:
493 # * start- and end-tag of a block must have exactly the same indentation.
494 # * start- and end-tags of *nested* blocks should have a greater indentation.
495 # NOTE: A single-line block must not start at beginning of the line with
496 # the same indentation as the enclosing multi-line blocks!
497 # Note that " " and "\t" are different, although they may
498 # look the same in an editor!
499 _s = re.escape(_block_start)
500 _e = re.escape(_block_end)
501 _strBlock = r"""
502 ^(?P<mEnd>[ \t]*)%send%s(?P<meIgnored>.*)\r?\n? # multi-line end (^ <!--(end)-->IGNORED_TEXT\n)
503 |
504 (?P<sEnd>)%send%s # single-line end (<!--(end)-->)
505 |
506 (?P<sSpace>[ \t]*) # single-line tag (no nesting)
507 %s(?P<sKeyw>\w+)[ \t]*(?P<sParam>.*?)%s
508 (?P<sContent>.*?)
509 (?=(?:%s.*?%s.*?)??%send%s) # (match until end or i.e. <!--(elif/else...)-->)
510 |
511 # multi-line tag, nested by whitespace indentation
512 ^(?P<indent>[ \t]*) # save indentation of start tag
513 %s(?P<mKeyw>\w+)\s*(?P<mParam>.*?)%s(?P<mIgnored>.*)\r?\n
514 (?P<mContent>(?:.*\n)*?)
515 (?=(?P=indent)%s(?:.|\s)*?%s) # match indentation
516 """ % (_s, _e,
517 _s, _e,
518 _s, _e, _s, _e, _s, _e,
519 _s, _e, _s, _e)
520 _reBlock = re.compile(_strBlock, re.X|re.M)
521
522 # "for"-block parameters: "var(,var)* in ..."
523 _strForParam = r"""^(?P<names>\w+(?:\s*,\s*\w+)*)\s+in\s+(?P<iter>.+)$"""
524 _reForParam = re.compile(_strForParam)
525
526 # allowed macro-names
527 _reMacroParam = re.compile(r"""^\w+$""")
528
529
530 def __init__(self, loadfunc=None, testexpr=None, escape=HTML):
531 """Init the parser.
532
533 :Parameters:
534 - `loadfunc`: function to load included templates
535 (i.e. ``LoaderFile(...).load``)
536 - `testexpr`: function to test if a template-expressions is valid
537 (i.e. ``EvalPseudoSandbox().compile``)
538 - `escape`: default-escaping (may be modified by the template)
539 :Exceptions:
540 - `ValueError`: if `testexpr` or `escape` is invalid.
541 """
542 if loadfunc is None:
543 self._load = dummy_raise(NotImplementedError, "'include' not supported, since no 'loadfunc' was given.")
544 else:
545 self._load = loadfunc
546
547 if testexpr is None:
548 self._testexprfunc = dummy
549 else:
550 try: # test if testexpr() works
551 testexpr("i==1")
552 except Exception as err:
553 raise ValueError("Invalid 'testexpr'. (%s)" %(err))
554 self._testexprfunc = testexpr
555
556 if escape not in ESCAPE_SUPPORTED.values():
557 raise ValueError("Unsupported 'escape'. (%s)" %(escape))
558 self.escape = escape
559 self._includestack = []
560
561 def parse(self, template):
562 """Parse a template.
563
564 :Parameters:
565 - `template`: template-unicode-string
566 :Returns: the resulting parse-tree
567 :Exceptions:
568 - `TemplateSyntaxError`: for template-syntax-errors
569 - `TemplateIncludeError`: if template-inclusion failed
570 - `TemplateException`
571 """
572 self._includestack = [(None, template)] # for error-messages (_errpos)
573 return self._parse(template)
574
575 def _errpos(self, fpos):
576 """Convert `fpos` to ``(filename,row,column)`` for error-messages."""
577 filename, string = self._includestack[-1]
578 return filename, srow(string, fpos), scol(string, fpos)
579
580 def _testexpr(self, expr, fpos=0):
581 """Test a template-expression to detect errors."""
582 try:
583 self._testexprfunc(expr)
584 except SyntaxError as err:
585 raise TemplateSyntaxError(err, self._errpos(fpos))
586
587 def _parse_sub(self, parsetree, text, fpos=0):
588 """Parse substitutions, and append them to the parse-tree.
589
590 Additionally, remove comments.
591 """
592 curr = 0
593 for match in self._reSubstitution.finditer(text):
594 start = match.start()
595 if start > curr:
596 parsetree.append(("str", self._reComment.sub('', text[curr:start])))
597
598 if match.group("sub") is not None:
599 if not match.group("end"):
600 raise TemplateSyntaxError("Missing closing tag '%s' for '%s'."
601 % (self._sub_end, match.group()), self._errpos(fpos+start))
602 if len(match.group("sub")) > 0:
603 self._testexpr(match.group("sub"), fpos+start)
604 parsetree.append(("sub", match.group("sub")))
605 else:
606 assert(match.group("escsub") is not None)
607 if not match.group("escend"):
608 raise TemplateSyntaxError("Missing closing tag '%s' for '%s'."
609 % (self._subesc_end, match.group()), self._errpos(fpos+start))
610 if len(match.group("escsub")) > 0:
611 self._testexpr(match.group("escsub"), fpos+start)
612 parsetree.append(("esc", self.escape, match.group("escsub")))
613
614 curr = match.end()
615
616 if len(text) > curr:
617 parsetree.append(("str", self._reComment.sub('', text[curr:])))
618
619 def _parse(self, template, fpos=0):
620 """Recursive part of `parse()`.
621
622 :Parameters:
623 - template
624 - fpos: position of ``template`` in the complete template (for error-messages)
625 """
626 # blank out comments
627 # (So that its content does not collide with other syntax, and
628 # because removing them completely would falsify the character-
629 # position ("match.start()") of error-messages)
630 template = self._reComment.sub(lambda match: self._comment_start+" "*len(match.group(1))+match.group(2), template)
631
632 # init parser
633 parsetree = []
634 curr = 0 # current position (= end of previous block)
635 block_type = None # block type: if,for,macro,raw,...
636 block_indent = None # None: single-line, >=0: multi-line
637
638 # find blocks
639 for match in self._reBlock.finditer(template):
640 start = match.start()
641 # process template-part before this block
642 if start > curr:
643 self._parse_sub(parsetree, template[curr:start], fpos)
644
645 # analyze block syntax (incl. error-checking and -messages)
646 keyword = None
647 block = match.groupdict()
648 pos__ = fpos + start # shortcut
649 if block["sKeyw"] is not None: # single-line block tag
650 block_indent = None
651 keyword = block["sKeyw"]
652 param = block["sParam"]
653 content = block["sContent"]
654 if block["sSpace"]: # restore spaces before start-tag
655 if len(parsetree) > 0 and parsetree[-1][0] == "str":
656 parsetree[-1] = ("str", parsetree[-1][1] + block["sSpace"])
657 else:
658 parsetree.append(("str", block["sSpace"]))
659 pos_p = fpos + match.start("sParam") # shortcuts
660 pos_c = fpos + match.start("sContent")
661 elif block["mKeyw"] is not None: # multi-line block tag
662 block_indent = len(block["indent"])
663 keyword = block["mKeyw"]
664 param = block["mParam"]
665 content = block["mContent"]
666 pos_p = fpos + match.start("mParam")
667 pos_c = fpos + match.start("mContent")
668 ignored = block["mIgnored"].strip()
669 if ignored and ignored != self._comment_start:
670 raise TemplateSyntaxError("No code allowed after block-tag.", self._errpos(fpos+match.start("mIgnored")))
671 elif block["mEnd"] is not None: # multi-line block end
672 if block_type is None:
673 raise TemplateSyntaxError("No block to end here/invalid indent.", self._errpos(pos__) )
674 if block_indent != len(block["mEnd"]):
675 raise TemplateSyntaxError("Invalid indent for end-tag.", self._errpos(pos__) )
676 ignored = block["meIgnored"].strip()
677 if ignored and ignored != self._comment_start:
678 raise TemplateSyntaxError("No code allowed after end-tag.", self._errpos(fpos+match.start("meIgnored")))
679 block_type = None
680 elif block["sEnd"] is not None: # single-line block end
681 if block_type is None:
682 raise TemplateSyntaxError("No block to end here/invalid indent.", self._errpos(pos__))
683 if block_indent is not None:
684 raise TemplateSyntaxError("Invalid indent for end-tag.", self._errpos(pos__))
685 block_type = None
686 else:
687 raise TemplateException("FATAL: Block regexp error. Please contact the author. (%s)" % match.group())
688
689 # analyze block content (mainly error-checking and -messages)
690 if keyword:
691 keyword = keyword.lower()
692 if 'for' == keyword:
693 if block_type is not None:
694 raise TemplateSyntaxError("Missing block-end-tag before new block at '%s'." %(match.group()), self._errpos(pos__))
695 block_type = 'for'
696 cond = self._reForParam.match(param)
697 if cond is None:
698 raise TemplateSyntaxError("Invalid 'for ...' at '%s'." %(param), self._errpos(pos_p))
699 names = tuple(n.strip() for n in cond.group("names").split(","))
700 self._testexpr(cond.group("iter"), pos_p+cond.start("iter"))
701 parsetree.append(("for", names, cond.group("iter"), self._parse(content, pos_c)))
702 elif 'if' == keyword:
703 if block_type is not None:
704 raise TemplateSyntaxError("Missing block-end-tag before new block at '%s'." %(match.group()), self._errpos(pos__))
705 if not param:
706 raise TemplateSyntaxError("Missing condition for 'if' at '%s'." %(match.group()), self._errpos(pos__))
707 block_type = 'if'
708 self._testexpr(param, pos_p)
709 parsetree.append(("if", param, self._parse(content, pos_c)))
710 elif 'elif' == keyword:
711 if block_type != 'if':
712 raise TemplateSyntaxError("'elif' may only appear after 'if' at '%s'." %(match.group()), self._errpos(pos__))
713 if not param:
714 raise TemplateSyntaxError("Missing condition for 'elif' at '%s'." %(match.group()), self._errpos(pos__))
715 self._testexpr(param, pos_p)
716 parsetree.append(("elif", param, self._parse(content, pos_c)))
717 elif 'else' == keyword:
718 if block_type not in ('if', 'for'):
719 raise TemplateSyntaxError("'else' may only appear after 'if' or 'for' at '%s'." %(match.group()), self._errpos(pos__))
720 if param:
721 raise TemplateSyntaxError("'else' may not have parameters at '%s'." %(match.group()), self._errpos(pos__))
722 parsetree.append(("else", self._parse(content, pos_c)))
723 elif 'macro' == keyword:
724 if block_type is not None:
725 raise TemplateSyntaxError("Missing block-end-tag before new block '%s'." %(match.group()), self._errpos(pos__))
726 block_type = 'macro'
727 # make sure param is "\w+" (instead of ".+")
728 if not param:
729 raise TemplateSyntaxError("Missing name for 'macro' at '%s'." %(match.group()), self._errpos(pos__))
730 if not self._reMacroParam.match(param):
731 raise TemplateSyntaxError("Invalid name for 'macro' at '%s'." %(match.group()), self._errpos(pos__))
732 #remove last newline
733 if len(content) > 0 and content[-1] == '\n':
734 content = content[:-1]
735 if len(content) > 0 and content[-1] == '\r':
736 content = content[:-1]
737 parsetree.append(("macro", param, self._parse(content, pos_c)))
738
739 # parser-commands
740 elif 'raw' == keyword:
741 if block_type is not None:
742 raise TemplateSyntaxError("Missing block-end-tag before new block '%s'." %(match.group()), self._errpos(pos__))
743 if param:
744 raise TemplateSyntaxError("'raw' may not have parameters at '%s'." %(match.group()), self._errpos(pos__))
745 block_type = 'raw'
746 parsetree.append(("str", content))
747 elif 'include' == keyword:
748 if block_type is not None:
749 raise TemplateSyntaxError("Missing block-end-tag before new block '%s'." %(match.group()), self._errpos(pos__))
750 if param:
751 raise TemplateSyntaxError("'include' may not have parameters at '%s'." %(match.group()), self._errpos(pos__))
752 block_type = 'include'
753 try:
754 u = self._load(content.strip())
755 except Exception as err:
756 raise TemplateIncludeError(err, self._errpos(pos__))
757 self._includestack.append((content.strip(), u)) # current filename/template for error-msg.
758 p = self._parse(u)
759 self._includestack.pop()
760 parsetree.extend(p)
761 elif 'set_escape' == keyword:
762 if block_type is not None:
763 raise TemplateSyntaxError("Missing block-end-tag before new block '%s'." %(match.group()), self._errpos(pos__))
764 if param:
765 raise TemplateSyntaxError("'set_escape' may not have parameters at '%s'." %(match.group()), self._errpos(pos__))
766 block_type = 'set_escape'
767 esc = content.strip().upper()
768 if esc not in ESCAPE_SUPPORTED:
769 raise TemplateSyntaxError("Unsupported escape '%s'." %(esc), self._errpos(pos__))
770 self.escape = ESCAPE_SUPPORTED[esc]
771 else:
772 raise TemplateSyntaxError("Invalid keyword '%s'." %(keyword), self._errpos(pos__))
773 curr = match.end()
774
775 if block_type is not None:
776 raise TemplateSyntaxError("Missing end-tag.", self._errpos(pos__))
777
778 if len(template) > curr: # process template-part after last block
779 self._parse_sub(parsetree, template[curr:], fpos+curr)
780
781 return parsetree
782
783#-----------------------------------------
784# Evaluation
785
786# some checks
787assert len(eval("dir()", {'__builtins__':{'dir':dir}})) == 1, \
788 "FATAL: 'eval' does not work as expected (%s)."
789assert compile("0 .__class__", "<string>", "eval").co_names == ('__class__',), \
790 "FATAL: 'compile' does not work as expected."
791
792class EvalPseudoSandbox:
793 """An eval-pseudo-sandbox.
794
795 The pseudo-sandbox restricts the available functions/objects, so the
796 code can only access:
797
798 - some of the builtin Python-functions, which are considered "safe"
799 (see safe_builtins)
800 - some additional functions (exists(), default(), setvar(), escape())
801 - the passed objects incl. their methods.
802
803 Additionally, names beginning with "_" are forbidden.
804 This is to prevent things like '0 .__class__', with which you could
805 easily break out of a "sandbox".
806
807 Be careful to only pass "safe" objects/functions to the template,
808 because any unsafe function/method could break the sandbox!
809 For maximum security, restrict the access to as few objects/functions
810 as possible!
811
812 :Warning:
813 Note that this is no real sandbox! (And although I don't know any
814 way to break out of the sandbox without passing-in an unsafe object,
815 I cannot guarantee that there is no such way. So use with care.)
816
817 Take care if you want to use it for untrusted code!!
818 """
819
820 safe_builtins = {
821 "True" : True,
822 "False" : False,
823 "None" : None,
824
825 "abs" : builtins.abs,
826 "chr" : builtins.chr,
827 "divmod" : builtins.divmod,
828 "hash" : builtins.hash,
829 "hex" : builtins.hex,
830 "isinstance": builtins.isinstance,
831 "len" : builtins.len,
832 "max" : builtins.max,
833 "min" : builtins.min,
834 "oct" : builtins.oct,
835 "ord" : builtins.ord,
836 "pow" : builtins.pow,
837 "range" : builtins.range,
838 "round" : builtins.round,
839 "sorted" : builtins.sorted,
840 "sum" : builtins.sum,
841 "unichr" : builtins.chr,
842 "zip" : builtins.zip,
843
844 "bool" : builtins.bool,
845 "bytes" : builtins.bytes,
846 "complex" : builtins.complex,
847 "dict" : builtins.dict,
848 "enumerate" : builtins.enumerate,
849 "float" : builtins.float,
850 "int" : builtins.int,
851 "list" : builtins.list,
852 "long" : long,
853 "reversed" : builtins.reversed,
854 "set" : builtins.set,
855 "str" : builtins.str,
856 "tuple" : builtins.tuple,
857 "unicode" : unicode,
858
859 "dir" : builtins.dir,
860 }
861 if sys.version_info[0] < 3:
862 safe_builtins["unichr"] = builtins.unichr
863
864 def __init__(self):
865 self._compile_cache = {}
866 self.vars_ptr = None
867 self.eval_allowed_builtins = self.safe_builtins.copy()
868 self.register("__import__", self.f_import)
869 self.register("exists", self.f_exists)
870 self.register("default", self.f_default)
871 self.register("setvar", self.f_setvar)
872 self.register("escape", self.f_escape)
873
874 def register(self, name, obj):
875 """Add an object to the "allowed eval-builtins".
876
877 Mainly useful to add user-defined functions to the pseudo-sandbox.
878 """
879 self.eval_allowed_builtins[name] = obj
880
881 def _check_code_names(self, code, expr):
882 """Check if the code tries to access names beginning with "_".
883
884 Used to prevent sandbox-breakouts via new-style-classes, like
885 ``"".__class__.__base__.__subclasses__()``.
886
887 :Raises:
888 NameError if expression contains forbidden names.
889 """
890 for name in code.co_names:
891 if name[0] == '_' and name != '_[1]': # _[1] is necessary for [x for x in y]
892 raise NameError("Name '%s' is not allowed in '%s'." % (name, expr))
893 # recursively check sub-codes (e.g. lambdas)
894 for const in code.co_consts:
895 if isinstance(const, types.CodeType):
896 self._check_code_names(const, expr)
897
898 def compile(self, expr):
899 """Compile a Python-eval-expression.
900
901 - Use a compile-cache.
902 - Raise a `NameError` if `expr` contains a name beginning with ``_``.
903
904 :Returns: the compiled `expr`
905 :Exceptions:
906 - `SyntaxError`: for compile-errors
907 - `NameError`: if expr contains a name beginning with ``_``
908 """
909 if expr not in self._compile_cache:
910 c = compile(expr, "", "eval")
911 self._check_code_names(c, expr)
912 self._compile_cache[expr] = c
913 return self._compile_cache[expr]
914
915 def eval(self, expr, variables):
916 """Eval a Python-eval-expression.
917
918 Sets ``self.vars_ptr`` to ``variables`` and compiles the code
919 before evaluating.
920 """
921 sav = self.vars_ptr
922 self.vars_ptr = variables
923
924 try:
925 x = eval(self.compile(expr), {"__builtins__": self.eval_allowed_builtins}, variables)
926 except NameError:
927 # workaround for lambdas like ``sorted(..., key=lambda x: my_f(x))``
928 vars2 = {"__builtins__": self.eval_allowed_builtins}
929 vars2.update(variables)
930 x = eval(self.compile(expr), vars2)
931
932 self.vars_ptr = sav
933 return x
934
935 def f_import(self, name, *_, **__):
936 """``import``/``__import__()`` for the sandboxed code.
937
938 Since "import" is insecure, the PseudoSandbox does not allow to
939 import other modules. But since some functions need to import
940 other modules (e.g. "datetime.datetime.strftime" imports "time"),
941 this function replaces the builtin "import" and allows to use
942 modules which are already accessible by the sandboxed code.
943
944 :Note:
945 - This probably only works for rather simple imports.
946 - For security, it may be better to avoid such (complex) modules
947 which import other modules. (e.g. use time.localtime and
948 time.strftime instead of datetime.datetime.strftime,
949 or write a small wrapper.)
950
951 :Example:
952
953 >>> from datetime import datetime
954 >>> import pyratemp
955 >>> t = pyratemp.Template('@!mytime.strftime("%H:%M:%S")!@')
956
957 # >>> print(t(mytime=datetime.now()))
958 # Traceback (most recent call last):
959 # ...
960 # ImportError: import not allowed in pseudo-sandbox; try to import 'time' yourself and pass it to the sandbox/template
961
962 >>> import time
963 >>> print(t(mytime=datetime.strptime("13:40:54", "%H:%M:%S"), time=time))
964 13:40:54
965
966 # >>> print(t(mytime=datetime.now(), time=time))
967 # 13:40:54
968 """
969 if self.vars_ptr is not None and name in self.vars_ptr and isinstance(self.vars_ptr[name], types.ModuleType):
970 return self.vars_ptr[name]
971 else:
972 raise ImportError("import not allowed in pseudo-sandbox; try to import '%s' yourself (and maybe pass it to the sandbox/template)" % name)
973
974 def f_exists(self, varname):
975 """``exists()`` for the sandboxed code.
976
977 Test if the variable `varname` exists in the current namespace.
978
979 This only works for single variable names. If you want to test
980 complicated expressions, use i.e. `default`.
981 (i.e. `default("expr",False)`)
982
983 :Note: the variable-name has to be quoted! (like in eval)
984 :Example: see module-docstring
985 """
986 return (varname in self.vars_ptr)
987
988 def f_default(self, expr, default=None):
989 """``default()`` for the sandboxed code.
990
991 Try to evaluate an expression and return the result or a
992 fallback-/default-value; the `default`-value is used
993 if `expr` does not exist/is invalid/results in None.
994
995 This is very useful for optional data.
996
997 :Parameter:
998 - expr: "eval-expression"
999 - default: fallback-value if eval(expr) fails or is None.
1000 :Returns:
1001 the eval-result or the "fallback"-value.
1002
1003 :Note: the eval-expression has to be quoted! (like in eval)
1004 :Example: see module-docstring
1005 """
1006 try:
1007 r = self.eval(expr, self.vars_ptr)
1008 if r is None:
1009 return default
1010 return r
1011 #TODO: which exceptions should be catched here?
1012 except (NameError, LookupError, TypeError, AttributeError):
1013 return default
1014
1015 def f_setvar(self, name, expr):
1016 """``setvar()`` for the sandboxed code.
1017
1018 Set a variable.
1019
1020 :Example: see module-docstring
1021 """
1022 self.vars_ptr[name] = self.eval(expr, self.vars_ptr)
1023 return ""
1024
1025 def f_escape(self, s, format="HTML"):
1026 """``escape()`` for the sandboxed code.
1027 """
1028 if isinstance(format, (str, unicode)):
1029 format = ESCAPE_SUPPORTED[format.upper()]
1030 return escape(unicode(s), format)
1031
1032#-----------------------------------------
1033# basic template / subtemplate
1034
1035class TemplateBase:
1036 """Basic template-class.
1037
1038 Used both for the template itself and for 'macro's ("subtemplates") in
1039 the template.
1040 """
1041
1042 def __init__(self, parsetree, renderfunc, data=None):
1043 """Create the Template/Subtemplate/Macro.
1044
1045 :Parameters:
1046 - `parsetree`: parse-tree of the template/subtemplate/macro
1047 - `renderfunc`: render-function
1048 - `data`: data to fill into the template by default (dictionary).
1049 This data may later be overridden when rendering the template.
1050 :Exceptions:
1051 - `TypeError`: if `data` is not a dictionary
1052 """
1053 #TODO: parameter-checking?
1054 self.parsetree = parsetree
1055 if isinstance(data, dict):
1056 self.data = data
1057 elif data is None:
1058 self.data = {}
1059 else:
1060 raise TypeError('"data" must be a dict (or None).')
1061 self.current_data = data
1062 self._render = renderfunc
1063
1064 def __call__(self, **override):
1065 """Fill out/render the template.
1066
1067 :Parameters:
1068 - `override`: objects to add to the data-namespace, overriding
1069 the "default"-data.
1070 :Returns: the filled template (in unicode)
1071 :Note: This is also called when invoking macros
1072 (i.e. ``$!mymacro()!$``).
1073 """
1074 self.current_data = self.data.copy()
1075 self.current_data.update(override)
1076 u = "".join(self._render(self.parsetree, self.current_data))
1077 self.current_data = self.data # restore current_data
1078 return _dontescape(u) # (see class _dontescape)
1079
1080 def __unicode__(self):
1081 """Alias for __call__()."""
1082 return self.__call__()
1083 def __str__(self):
1084 """Alias for __call__()."""
1085 return self.__call__()
1086
1087#-----------------------------------------
1088# Renderer
1089
1090class _dontescape(unicode):
1091 """Unicode-string which should not be escaped.
1092
1093 If ``isinstance(object,_dontescape)``, then don't escape the object in
1094 ``@!...!@``. It's useful for not double-escaping macros, and it's
1095 automatically used for macros/subtemplates.
1096
1097 :Note: This only works if the object is used on its own in ``@!...!@``.
1098 It i.e. does not work in ``@!object*2!@`` or ``@!object + "hi"!@``.
1099 """
1100 __slots__ = []
1101
1102
1103class Renderer(object):
1104 """Render a template-parse-tree.
1105
1106 :Uses: `TemplateBase` for macros
1107 """
1108
1109 def __init__(self, evalfunc, escapefunc):
1110 """Init the renderer.
1111
1112 :Parameters:
1113 - `evalfunc`: function for template-expression-evaluation
1114 (i.e. ``EvalPseudoSandbox().eval``)
1115 - `escapefunc`: function for escaping special characters
1116 (i.e. `escape`)
1117 """
1118 #TODO: test evalfunc
1119 self.evalfunc = evalfunc
1120 self.escapefunc = escapefunc
1121
1122 def _eval(self, expr, data):
1123 """evalfunc with error-messages"""
1124 try:
1125 return self.evalfunc(expr, data)
1126 #TODO: any other errors to catch here?
1127 except (TypeError, NameError, LookupError, AttributeError, SyntaxError) as err:
1128 raise TemplateRenderError("Cannot eval expression '%s'. (%s: %s)" %(expr, err.__class__.__name__, err))
1129
1130 def render(self, parsetree, data):
1131 """Render a parse-tree of a template.
1132
1133 :Parameters:
1134 - `parsetree`: the parse-tree
1135 - `data`: the data to fill into the template (dictionary)
1136 :Returns: the rendered output-unicode-string
1137 :Exceptions:
1138 - `TemplateRenderError`
1139 """
1140 _eval = self._eval # shortcut
1141 output = []
1142 do_else = False # use else/elif-branch?
1143
1144 if parsetree is None:
1145 return ""
1146 for elem in parsetree:
1147 if "str" == elem[0]:
1148 output.append(elem[1])
1149 elif "sub" == elem[0]:
1150 output.append(unicode(_eval(elem[1], data)))
1151 elif "esc" == elem[0]:
1152 obj = _eval(elem[2], data)
1153 #prevent double-escape
1154 if isinstance(obj, _dontescape) or isinstance(obj, TemplateBase):
1155 output.append(unicode(obj))
1156 else:
1157 output.append(self.escapefunc(unicode(obj), elem[1]))
1158 elif "for" == elem[0]:
1159 do_else = True
1160 (names, iterable) = elem[1:3]
1161 try:
1162 loop_iter = iter(_eval(iterable, data))
1163 except TypeError:
1164 raise TemplateRenderError("Cannot loop over '%s'." % iterable)
1165 for i in loop_iter:
1166 do_else = False
1167 if len(names) == 1:
1168 data[names[0]] = i
1169 else:
1170 data.update(zip(names, i)) #"for a,b,.. in list"
1171 output.extend(self.render(elem[3], data))
1172 elif "if" == elem[0]:
1173 do_else = True
1174 if _eval(elem[1], data):
1175 do_else = False
1176 output.extend(self.render(elem[2], data))
1177 elif "elif" == elem[0]:
1178 if do_else and _eval(elem[1], data):
1179 do_else = False
1180 output.extend(self.render(elem[2], data))
1181 elif "else" == elem[0]:
1182 if do_else:
1183 do_else = False
1184 output.extend(self.render(elem[1], data))
1185 elif "macro" == elem[0]:
1186 data[elem[1]] = TemplateBase(elem[2], self.render, data)
1187 else:
1188 raise TemplateRenderError("Invalid parse-tree (%s)." %(elem))
1189
1190 return output
1191
1192#-----------------------------------------
1193# template user-interface (putting it all together)
1194
1195class Template(TemplateBase):
1196 """Template-User-Interface.
1197
1198 :Usage:
1199 ::
1200 t = Template(...) (<- see __init__)
1201 output = t(...) (<- see TemplateBase.__call__)
1202
1203 :Example:
1204 see module-docstring
1205 """
1206
1207 def __init__(self, string=None,filename=None,parsetree=None, encoding='utf-8', data=None, escape=HTML,
1208 loader_class=LoaderFile,
1209 parser_class=Parser,
1210 renderer_class=Renderer,
1211 eval_class=EvalPseudoSandbox,
1212 escape_func=escape):
1213 """Load (+parse) a template.
1214
1215 :Parameters:
1216 - `string,filename,parsetree`: a template-string,
1217 filename of a template to load,
1218 or a template-parsetree.
1219 (only one of these 3 is allowed)
1220 - `encoding`: encoding of the template-files (only used for "filename")
1221 - `data`: data to fill into the template by default (dictionary).
1222 This data may later be overridden when rendering the template.
1223 - `escape`: default-escaping for the template, may be overwritten by the template!
1224 - `loader_class`
1225 - `parser_class`
1226 - `renderer_class`
1227 - `eval_class`
1228 - `escapefunc`
1229 """
1230 if [string, filename, parsetree].count(None) != 2:
1231 raise ValueError('Exactly 1 of string,filename,parsetree is necessary.')
1232
1233 tmpl = None
1234 # load template
1235 if filename is not None:
1236 incl_load = loader_class(os.path.dirname(filename), encoding).load
1237 tmpl = incl_load(os.path.basename(filename))
1238 if string is not None:
1239 incl_load = dummy_raise(NotImplementedError, "'include' not supported for template-strings.")
1240 tmpl = LoaderString(encoding).load(string)
1241
1242 # eval (incl. compile-cache)
1243 templateeval = eval_class()
1244
1245 # parse
1246 if tmpl is not None:
1247 p = parser_class(loadfunc=incl_load, testexpr=templateeval.compile, escape=escape)
1248 parsetree = p.parse(tmpl)
1249 del p
1250
1251 # renderer
1252 renderfunc = renderer_class(templateeval.eval, escape_func).render
1253
1254 #create template
1255 TemplateBase.__init__(self, parsetree, renderfunc, data)
1256
1257
1258#=========================================