defscan_short_string(self):
m = re.match(Lexer.re_short_string, self.chunk)if m:
s = m.group()
self.next(len(s))
s = s[1:len(s)-1]if s.find('\\')>=0:
result = Lexer.re_new_line.findall(s)
self.line +=len(result)
s = self.escape(s)return s
self.error('unfinished string')return''defescape(self, s):
ret =''whilelen(s)>0:if s[0]!='\\':
ret += s[0]
s = s[1:]continueiflen(s)==1:
self.error('unfinished string')if s[1]=='a':
ret +='\a'
s = s[2:]continueelif s[1]=='b':
ret +='\b'
s = s[2:]continueelif s[1]=='f':
ret +='\f'
s = s[2:]continueelif s[1]=='n'or s[1]=='\n':
ret +='\n'
s = s[2:]continueelif s[1]=='r':
ret +='\r'
s = s[2:]continueelif s[1]=='t':
ret +='\t'
s = s[2:]continueelif s[1]=='v':
ret +='\v'
s = s[2:]continueelif s[1]=='"':
ret +='"'
s = s[2:]continueelif s[1]=='\'':
ret +='\''
s = s[2:]continueelif s[1]=='\\':
ret +='\\'
s = s[2:]continueelif s[1]in'0123456789':
m = re.match(Lexer.re_dec_escape_seq, s)if m:
str_dec = m.group()[1:]
d =int(str_dec)if d <=0xff:
ret +=str(chr(d))
s = s[len(m.group()):]continue
self.error('decimal escape too large near "%s"'% m.group())elif s[1]=='x':
m = re.match(Lexer.re_hex_escape_seq, s)if m:
str_hex ='0'+ m.group()[1:]
d =int(str_hex,16)
ret +=str(chr(d))
s = s[len(m.group()):]continueelif s[1]=='u':
m = re.match(Lexer.re_unicode_escape_seq, s)if m:
str_unicode = m.group()[3:len(m.group())-1]
d =int(str_unicode,16)if d <=0x10ffff:
ret +=str(chr(d))
s = s[len(m.group()):]continue
self.error('UTF-8 value too large near "%s"'% str_unicode)elif s[1]=='z':
s = s[2:]whilelen(s)>0and Lexer.is_white_space(s[0]):
s = s[1:]continuereturn ret
long string
defscan_long_string(self):
m_open = re.match(Lexer.re_opening_long_bracket, self.chunk)if m_open isNone:
self.error('invalid long string delimiter near "%s"'% self.chunk[0:2])
str_open = m_open.group()
str_close = str_open.replace('[',']')
close_idx = self.chunk.find(str_close)if close_idx <0:
self.error('unfinished long string or comment')
s = self.chunk[len(str_open): close_idx]
self.next(close_idx +len(str_close))
s = re.sub(Lexer.re_new_line, s,'\n')
self.line += s.count('\n')iflen(s)>0and s[0]=='\n':
s = s[1:]return s
print("hello") -- short comment
print("world") --> another short comment
print() --[[ long comment ]]
--[===[
another
long comment
]===]
print("hello, \z
world!") --> hello, world!
a = 'alo\n123"'
a = "alo\n123\""
a = '\97lo\10\04923"'
a = [[alo
123"]]
a = [==[
alo
123"]==]
a = 123.456e78
a = 'ação'
a = '\x66'
a = '\u{03a9}\u{03a8}'
a = [======aaa
import sys
from lexer import Lexer
from lua_token import*defmain():withopen(sys.argv[1],'r')as f:
data = f.read()
lexer = Lexer(data, sys.argv[1])whileTrue:try:
line, kind, token = lexer.get_next_token()print('[%2d] [%-10s] %s'%(line, kind_to_category(kind), token))if kind == TokenKind.EOF:breakexcept Exception as e:
sys.exit(e)if __name__ =='__main__':iflen(sys.argv)==2:
main()else:print('Error argument')
result
[ 1][identifier] print
[ 1][separator ]([ 1][string ] hello
[ 1][separator ])[ 2][identifier] print
[ 2][separator ]([ 2][string ] world
[ 2][separator ])[ 3][identifier] print
[ 3][separator ]([ 3][separator ])[ 9][identifier] print
[ 9][separator ]([ 9][string ] hello, world![10][separator ])[12][identifier] a
[12][operator ]=[12][string ] alo
123"
[13] [identifier] a
[13] [operator ] =
[13] [string ] alo
123"[14][identifier] a
[14][operator ]=[14][string ] alo
123"
[15] [identifier] a
[15] [operator ] =
[15] [string ] alo
123"[17][identifier] a
[17][operator ]=[17][string ] alo
123"
[20] [identifier] a
[20] [operator ] =
[20] [number ] 123.456e78
[21] [identifier] a
[21] [operator ] =
[21] [string ] ação
[22] [identifier] a
[22] [operator ] =
[22] [string ] f
[23] [identifier] a
[23] [operator ] =
[23] [string ] ΩΨ
[24] [identifier] a
[24] [operator ] =
./test/lexer.lua:24: invalid long string delimiter near "[="