summaryrefslogtreecommitdiff
path: root/src/bin/elua/modules/xgettext/lexer.lua
blob: 0c8ebd5d11a0cf41655ede3b50adf70098b3744f (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
-- Elua xgettext: lexer

local yield = coroutine.yield
local tconc = table.concat

local keywords = {
    ["and"     ] = true, ["break" ] = true, ["do"   ] = true, ["else"] = true,
    ["elseif"  ] = true, ["end"   ] = true, ["false"] = true, ["for" ] = true,
    ["function"] = true, ["goto"  ] = true, ["if"   ] = true, ["in"  ] = true,
    ["local"   ] = true, ["nil"   ] = true, ["not"  ] = true, ["or"  ] = true,
    ["repeat"  ] = true, ["return"] = true, ["then" ] = true, ["true"] = true,
    ["until"   ] = true, ["while" ] = true
}

local tokens = {
    "..", "...", "==", ">=", "<=", "~=", "::",
    "<name>", "<string>", "<number>", "<eof>"
}

local max_custom_len = 79
local max_fname_len = 72
local max_str_len = 63

local source_to_msg = function(source)
    local c = source:sub(1, 1)
    local srclen = #source
    if c == "@" then
        if srclen <= (max_fname_len + 1) then
            return source:sub(2)
        else
            return "..." .. source:sub(srclen - max_fname_len + 1)
        end
    elseif c == "=" then
        return source:sub(2, max_custom_len + 1)
    else
        return '[string "' .. source:sub(1, max_str_len)
            .. ((srclen > max_str_len) and '..."]' or '"]')
    end
end

local lex_error = function(ls, msg, tok)
    msg = ("%s:%d: %s"):format(source_to_msg(ls.source), ls.line_number, msg)
    if tok then
        msg = msg .. " near '" .. tok .. "'"
    end
    error(msg, 0)
end

local syntax_error = function(ls, msg)
    lex_error(ls, msg, ls.token.value or ls.token.name)
end

local next_char = function(ls)
    local c = ls.reader()
    ls.current = c
    return c
end

local next_line = function(ls, cs)
    local old = ls.current
    assert(old == "\n" or old == "\r")
    local c = next_char(ls)
    if (c == "\n" or c == "\r") and c ~= old then
        c = next_char(ls)
    end
    ls.line_number = ls.line_number + 1
    return c
end

local read_number = function(ls, beg)
    local buf = {}
    if beg then buf[1] = beg end
    local c = ls.current
    while c == "." or c:match("%d") do
        buf[#buf + 1] = c
        c = next_char(ls)
    end
    if c == "e" or c == "E" then
        buf[#buf + 1] = c
        c = next_char(ls)
        if c == "+" or c == "-" then
            buf[#buf + 1] = c
            c = next_char(ls)
        end
    end
    while c:match("%w") do
        buf[#buf + 1] = c
        c = next_char(ls)
    end
    local str = tconc(buf)
    if not tonumber(str) then
        lex_error(ls, "malformed number", str)
    end
    return str
end

local skip_sep = function(ls, buf)
    local cnt = 0
    local s = ls.current
    assert(s == "[" or s == "]")
    buf[#buf + 1] = s
    local c = next_char(ls)
    while c == "=" do
        buf[#buf + 1] = c
        c = next_char(ls)
        cnt = cnt + 1
    end
    return c == s and cnt or ((-cnt) - 1)
end

local read_long_string = function(ls, sep, cmt)
    local buf = {}
    local c = next_char(ls)
    if c == "\n" or c == "\r" then c = next_line(ls) end
    while true do
        if not c then
            lex_error(ls, tok and cmt and "unfinished long comment"
                or "unfinished long string", "<eof>")
        elseif c == "]" then
            local tbuf = {}
            if skip_sep(ls, tbuf) == sep then
                c = next_char(ls)
                break
            else
                buf[#buf + 1] = tconc(tbuf)
            end
            c = ls.current
        else
            buf[#buf + 1] = c
            c = next_char(ls)
        end
    end
    return tconc(buf)
end

local read_string = function(ls)
    local delim = ls.current
    local buf = {}
    local c = next_char(ls)
    while c ~= delim do
        if not c then lex_error(ls, "unfinished string", "<eof>")
        elseif c == "\n" or c == "\r" then
            lex_error(ls, "unfinished string", tconc(buf))
        elseif c == "\\" then
            c = next_char(ls)
            if c == "n" then
                buf[#buf + 1] = "\n"
            else
                buf[#buf + 1] = "\\" .. c
            end
            c = next_char(ls)
        else
            buf[#buf + 1] = c
            c = next_char(ls)
        end
    end
    next_char(ls)
    return tconc(buf)
end

local last_comment = false

local match_comment = function(ls, cmt)
    cmt = cmt:match("^%s*(.+)%s*$")
    if ls.flags[cmt] then
        return "<flagcomment>", cmt
    end
    local   lcmt = ls.lex_cmt
    if not  lcmt then return nil end
    if type(lcmt) ~= "string" then
        return "<comment>", cmt
    end
    lcmt = lcmt:match("^%s*(.+)%s*$")
    if last_comment or cmt:sub(1, #lcmt) == lcmt then
        return "<comment>", cmt
    end
    return nil
end

local lex_tbl = {
    ["\n"] = function(ls) next_line(ls) end,
    [" " ] = function(ls) next_char(ls) end,
    ["-" ] = function(ls)
        local c = next_char(ls)
        if c ~= "-" then
            return "-"
        end
        c = next_char(ls)
        if c == "[" then
            local sep = skip_sep(ls, {})
            if sep >= 0 then
                return match_comment(ls, read_long_string(ls, sep, true))
            end
        end
        local buf = {}
        while ls.current and ls.current ~= "\n" and ls.current ~= "\r" do
            buf[#buf + 1] = ls.current
            next_char(ls)
        end
        return match_comment(ls, tconc(buf))
    end,
    ["[" ] = function(ls)
        local buf = {}
        local sep = skip_sep(ls, {})
        if sep >= 0 then
            return "<string>", read_long_string(ls, sep)
        elseif sep == -1 then
            return "["
        else
            lex_error(ls, "invalid long string delimiter", tconc(buf))
        end
    end,
    ["="] = function(ls)
        local oc = ls.current
        local c  = next_char(ls)
        if c ~= "=" then return c
        else next_char(ls); return c .. "=" end
    end,
    ['"' ] = function(ls)
        return "<string>", read_string(ls)
    end,
    ["." ] = function(ls)
        local c = next_char(ls)
        if c == "." then
            c = next_char(ls)
            if c == "." then
                next_char(ls)
                return "..."
            else
                return ".."
            end
        elseif c:match("%d") then
            return "<number>", read_number(ls, ".")
        else
            return "."
        end
    end,
    ["0" ] = function(ls)
        return "<number>", read_number(ls)
    end
}
lex_tbl["\r"] = lex_tbl["\n"]
lex_tbl["\f"] = lex_tbl[" " ]
lex_tbl["\t"] = lex_tbl[" " ]
lex_tbl["\v"] = lex_tbl[" " ]
lex_tbl["<" ] = lex_tbl["=" ]
lex_tbl[">" ] = lex_tbl["=" ]
lex_tbl["~" ] = lex_tbl["=" ]
lex_tbl["'" ] = lex_tbl['"' ]
lex_tbl["1" ] = lex_tbl["0" ]
lex_tbl["2" ] = lex_tbl["0" ]
lex_tbl["3" ] = lex_tbl["0" ]
lex_tbl["4" ] = lex_tbl["0" ]
lex_tbl["5" ] = lex_tbl["0" ]
lex_tbl["6" ] = lex_tbl["0" ]
lex_tbl["7" ] = lex_tbl["0" ]
lex_tbl["8" ] = lex_tbl["0" ]
lex_tbl["9" ] = lex_tbl["0" ]

local lex_default = function(ls)
    local c = ls.current
    if c == "_" or c:match("%a") then
        local buf = {}
        repeat
            buf[#buf + 1] = c
            c = next_char(ls)
            if not c then break end
        until not (c == "_" or c:match("%w"))
        local str = tconc(buf)
        if keywords[str] then
            return str
        end
        return "<name>", str
    else
        next_char(ls)
        return c
    end
end

local lex_main = function(ls)
    yield()
    while true do
        local c = ls.current
        if c == nil then
            return "<eof>"
        end
        local opt = lex_tbl[c]
        if opt then
            local t, v = opt(ls)
            if t then
                last_comment = t == "<comment>"
                yield(t, v)
            end
        else
            last_comment = false
            yield(lex_default(ls))
        end
    end
end

local strstream = function(str)
    return str:gmatch(".")
end

local skip_bom = function(rdr)
    local c = rdr()
    if c ~= 0xEF then return c end
    c = rdr()
    if c ~= 0xBB then return c end
    c = rdr()
    if c ~= 0xBF then return c end
    return rdr()
end

local skip_shebang = function(rdr)
    local c = skip_bom(rdr)
    if c == 35 then -- #
        repeat
            c = rdr()
        until not c or is_newline(c)
        local e = c
        c = rdr()
        if (e == 10 and c == 13) or (e == 13 and c == 10) then -- LF, CR
            c = rdr()
        end
    end
    return c
end

local ls_get = function(self)
    local tok = self.token
    tok.name, tok.value = self.coro()
    return tok
end

return { init = function(chunkname, input, flags, opts)
    local reader = type(input) == "string" and strstream(input) or input
    local current = skip_shebang(reader)
    local ls = {
        reader      = reader,
        token       = {},
        source      = chunkname,
        current     = current,
        line_number = 1,
        get         = ls_get,
        lex_cmt     = opts["c"],
        flags       = flags.valid
    }
    local coro = coroutine.wrap(lex_main, ls)
    ls.coro = coro
    coro(ls)
    return ls
end, syntax_error = syntax_error, source_to_msg = source_to_msg }