Skip to content

misc: add lua-parser-loose-scm-1.rockspec #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

misc: add lua-parser-loose-scm-1.rockspec #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f3d5901
misc: add lua-parser-loose-scm-1.rockspec
Oct 17, 2014
67f9007
Paul Kulchenko's fork
Oct 27, 2014
c6f3f7b
update @pkulchenko 's sources (no commit history)
Oct 29, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 31 additions & 57 deletions lib/lua_lexer_loose.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
lua_lexer_loose.lua.
Loose lexing of Lua code. See README.

WARNING: This code is preliminary and may have errors
in its current form.

(c) 2013 David Manura. MIT License.
--]]

Expand All @@ -16,7 +19,7 @@ local function match_string(s, pos)
pos = pos + 1
while 1 do
pos = s:find("[" .. c .. "\\]", pos)
if not pos then return nil, posa, 'syntax error' end
if not pos then return s:sub(posa), #s + 1 end -- not terminated string
if s:sub(pos,pos) == c then
local part = s:sub(posa, pos)
return part, pos + 1
Expand All @@ -28,7 +31,7 @@ local function match_string(s, pos)
local sc = s:match("^%[(=*)%[", pos)
if sc then
local _; _, pos = s:find("%]" .. sc .. "%]", pos)
if not pos then return nil, posa, 'syntax error' end
if not pos then return s:sub(posa), #s + 1 end -- not terminated string
local part = s:sub(posa, pos)
return part, pos + 1
else
Expand Down Expand Up @@ -56,40 +59,16 @@ end

-- note: matches invalid numbers too
local function match_numberlike(s, pos)
local a,b = s:match('^(%.?)([0-9])', pos)
if not a then
return nil -- not number
end
local tok, more
if b == '0' then
tok, more = s:match('^(0[xX][0-9a-fA-F]*)([_g-zG-Z]?)', pos)
if tok then -- hex
if #more == 0 and #tok > 2 then return tok end
end
end
if a == '.' then
tok, more = s:match('^(%.[0-9]+)([a-zA-Z_%.]?)', pos)
else
tok, more = s:match('^([0-9]+%.?[0-9]*)([a-zA-Z_%.]?)', pos)
end
if more ~= '' then
if more == 'e' or more == 'E' then -- exponent
local tok2, bad = s:match('^([eE][+-]?[0-9]+)([_a-zA-Z]?)', pos + #tok)
if tok2 and bad == '' then
return tok..tok2
else
local tok2 = assert(s:match('^[eE][+-]?[0-9a-zA-Z_]*', pos + #tok))
return tok..tok2, 'bad number'
end
else
local tok2 = s:match('^[0-9a-zA-Z_%.]*', pos + #tok)
return tok..tok2, 'bad number'
end
local tok = s:match('^0[xX][0-9A-Fa-f]*', pos)
if tok then return tok end
local tok = s:match('^[0-9%.]+', pos)
if tok then
local tok2 = s:match('^[eE][+-]?[0-9]*', pos + #tok)
if tok2 then tok = tok .. tok2 end
return tok
end
assert(tok)
return tok
return nil
end
--TODO: Lua 5.2 hex float

local function newset(s)
local t = {}
Expand All @@ -104,8 +83,8 @@ end

local sym = newset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_")
local dig = newset('0123456789')
local dig2 = qws[[.0 .1 .2 .3 .4 .5 .6 .7 .8 .9]]
local op = newset('=~<>.+-*/%^#=<>;:,.{}[]()')

op['=='] = true
op['<='] = true
op['>='] = true
Expand All @@ -117,13 +96,10 @@ local is_keyword = qws[[
in local nil not or repeat return
then true until while]]

function M.lex(code, f)
local pos = 1
local tok = code:match('^#[^\n]*\n?', pos) -- shebang
if tok then
--f('Shebang', tok, 1)
pos = pos + #tok
end
function M.lex(code, f, pos)
local pos = pos or 1
local tok = code:match('^#![^\n]*\n', pos) -- shebang
if tok then f('Shebang', tok, 1) pos = pos + #tok end
while pos <= #code do
local p2, n2, n1 = code:match('^%s*()((%S)%S?)', pos)
if not p2 then assert(code:sub(pos):match('^%s*$')); break end
Expand All @@ -144,18 +120,18 @@ function M.lex(code, f)
f('Comment', tok, pos)
pos = pos2
elseif n1 == '\'' or n1 == '\"' or n2 == '[[' or n2 == '[=' then
local tok, _pos2 = match_string(code, pos)
local tok = match_string(code, pos)
if tok then
f('String', tok, pos)
pos = pos + #tok
else
f('Unknown', code:sub(pos), pos) -- unterminated string
pos = #code + 1
end
pos = pos + #tok
elseif dig[n1] or dig2[n2] then
local tok, err = match_numberlike(code, pos) assert(tok)
elseif dig[n1] then
local tok = match_numberlike(code, pos)
assert(tok)
if err then f('Unknown', tok)
else f('Number', tok, pos) end
f('Number', tok, pos)
pos = pos + #tok
elseif op[n2] then
if n2 == '..' and code:match('^%.', pos+2) then
Expand All @@ -178,7 +154,7 @@ end

local Stream = {}
Stream.__index = Stream
function Stream:next()
function Stream:next(val)
if self._next then
local _next = self._next
self._next = nil
Expand All @@ -198,17 +174,15 @@ function Stream:peek()
end
end

function M.lexc(code, f)
function M.lexc(code, f, pos)
local yield = coroutine.yield
local f = coroutine.wrap(function()
M.lex(code, function(tag, name, pos) --print(tag, '['..name..']')
if tag ~= 'Comment' then
yield {tag=tag, name, lineinfo=pos}
end
end)
local func = coroutine.wrap(f or function()
M.lex(code, function(tag, name, pos)
yield {tag=tag, name, lineinfo=pos}
end, pos)
yield {tag='Eof'}
end)
return setmetatable({f=f}, Stream)
return setmetatable({f=func}, Stream)
end

return M
Loading