python表达式词法分析包_python实现词法分析

#请先安装Ply

# -*- coding: utf-8 -*-

#--------------------------------------------------------------------------

#Author:Jmdebugger

#email: pengkailb@gmail.com

#date: 2013-9-17

#--------------------------------------------------------------------------

import ply.lex as lex

tokens = [

"TOKEN_IDENT",

"TOKEN_INT",

"TOKEN_FLOAT",

"TOKEN_STRING",

"TOKEN_OP",

"TOKEN_DELIM_COMMA", #,

"TOKEN_DELIM_OPEN_PAREN", #(

"TOKEN_DELIM_CLOSE_PAREN", #)

"TOKEN_DELIM_OPEN_BRACKET", #[

"TOKEN_DELIM_CLOSE_BRACKET",#]

"TOKEN_DELIM_OPEN_BRACE", #{

"TOKEN_DELIM_CLOSE_BRACE", #}

"TOKEN_DELIM_SEMICOLON" #;

]

reserved = {

'if' : 'TOKEN_RSRVD_IF',

'else' : 'TOKEN_RSRVD_ELSE',

'true' : 'TOKEN_RSRVD_TRUE',

'false' : 'TOKEN_RSRVD_FALSE',

'while' : 'TOKEN_RSRVD_WHILE',

'break' : 'TOKEN_RSRVD_BREAK',

'continue': 'TOKEN_RSRVD_CONTINUE',

'goto' : 'TOKEN_RSRVD_GOTO',

'func' : 'TOKEN_RSRVD_FUNC',

'var' : 'TOKEN_RSRVD_VAR',

'for' : 'TOKEN_RSRVD_FOR',

'return' : 'TOKEN_RSRVD_RETURN'

}

tokens += reserved .values()

t_ignore = r' \t\r'

def t_COMMENT(t):

r'(/\*(.|\n)*?\*/)|(\/\/.*)'

pass

def t_newline(t):

r'\n+'

t.lexer.lineno += len(t.value)

def t_error(t):

print "LaunchScript error: "+repr(t.value)

def t_TOKEN_IDENT(t):

r'[a-zA-Z_][a-zA-Z_0-9]*' #标识符

t.type = reserved.get(t.value , 'TOKEN_IDENT')

return t

def t_TOKEN_INT(t):

r'(0x[a-fA-F0-9]+)|([0-9]+)'

return t

t_TOKEN_FLOAT = r'[0-9]*\.[0-9]+'

t_TOKEN_STRING = r'(\"([^\\\r]|(\\.))*?\")' #|(\"([^\\\n]|(\\.))*?\")' only for windows

t_TOKEN_DELIM_COMMA = r'\,'

t_TOKEN_DELIM_OPEN_PAREN = r'\('

t_TOKEN_DELIM_CLOSE_PAREN = r'\)'

t_TOKEN_DELIM_OPEN_BRACKET = r'\['

t_TOKEN_DELIM_CLOSE_BRACKET = r'\]'

t_TOKEN_DELIM_OPEN_BRACE = r'\{'

t_TOKEN_DELIM_CLOSE_BRACE = r'\}'

t_TOKEN_DELIM_SEMICOLON = r'\;'

def t_TOKEN_OP(t):

r'(\\>\=)|([\+\-\*\/\%\&\|\^\=\!\>\\

return t

if __name__ == "__main__":

lexer = lex.lex()

f = open("./test.txt" , 'rb')

data = f.read()

f.close()

lexer.input(data)

while True:

tok = lexer.token()

if not tok: break # No more input

print tok.value+"\t---->\t"+tok.type