python实现c语言测试,Python实现的C语言词法分析

编译原理课上的一个实验是做一个编译器前端的词法分析器，我选择了用Python来写C语言的词法分析。

词法分析器的功能是输入源程序，输出单词符号。当初定义Token(单词种别，属性值)序列的时候，是将单词种别用数字来表示，后来再做语法分析的时候，发现用数字时不太合理的，所以又对单词的种别码进行了一番修改。

我的程序的总体思路是先对源程序进行一遍扫描，将多余的空格和注释去除，然后再读一遍已经进行过预处理的源程序，进行单词的识别，转换成二元组，保存到token文件中，并建立符号表对标识符进行管理，如果发现了错误，对其的位置和错误信息进行打印。

在对单词的识别部分，我采用了有穷自动机的理论来进行识别。这样就可以根据现在的状态和输入符号决定其后继行为。因此在对单词的识别中，我画了很多的状态图来识别不同的单词，如字符串、数字等等。状态图的绘制中，本来想用visio来画的，后来的后来觉得太麻烦了，还是用了最快的手画的方法。

图1.注释的状态转换图

图2.标志符的状态装换图

图3.字符串的状态装换图

图4.界符的状态转换图

图5.整常数、浮点常数的状态转换图

图6.字符常数的状态转换图

关于错误处理的方面，我对于词法分析阶段所能遇到的几种错误，如下图所示中的四种中的前三种都进行了相应的处理。但是对于第三点做的不太好，对字符常数中可以出现的字符限制的有点过于厉害，例如分号等在我的词法分析器中是不能再字符串中出现的。

图7.词法分析中的四种错误

测试程序如下，内包含主要的C语言的各种语句，含有少量的错误：

int main()

{

int _a;

char ch = 'f;

floatb,centigrade,fahrj@enheit;

char fd = '\n';

printf("please inputa);

scanf("%d",&a); /*mycomment1or***2*/

printf("please inputb");

scanf("%f",&b);

if (a==8.1.6)

{

centigrade=095*(b-32)/9; /*itismyc5435omment*/

printf("TheCentigrade is ",centigrade); /*mess/age*/

}

else if (a!=0)

{

fahrenheit=(9/5.0)*b++32; /*mycontent*/

printf("TheFahrenheit is fahrenheit); /*hello****/

}

return 0;

}

运行结果如下图所示：

图8.测试程序的错误报告

这是用Python写的第一个稍微像点样的东西，所以很多地方写的不大好，代码结构也是有点混乱。总而言之，就是在这样的条件下把编译原理的第一次实验给写完了。接下来是我的水水的代码了。

# -*- coding: utf-8 -*-

'''

Created on 2012-10-18

@author: zouliping

'''

import string

_key = ("auto","break","case","char","const","continue","default",

"do","double","else","enum","extern","float","for",

"goto","if","int","long","register","return","short",

"signed","static","sizeof","struct","switch","typedef","union",

"unsigned","void","volatile","while") # c语言的32个关键字

_abnormalChar = '@#$%^&*~' #标识符中可能出现的非法字符

_syn = '' #单词的种别码

_p = 0 #下标

_value = '' #存放词法分析出的单词

_content = '' #程序内容

_mstate = 0 #字符串的状态

_cstate = 0 #字符的状态

_dstate = 0 #整数和浮点数的状态

_line = 1 #代码的第几行

_mysymbol = [] #符号表

def outOfComment():

'''去除代码中的注释'''

global _content

state = 0

index = -1

for c in _content:

index = index + 1

if state == 0:

if c == '/':

state = 1

startIndex = index

elif state == 1:

if c == '*':

state = 2

else:

state = 0

elif state == 2:

if c == '*':

state = 3

else:

pass

elif state == 3:

if c == '/':

endIndex = index + 1

comment = _content[startIndex:endIndex]

_content = _content.replace(comment,'') #将注释替换为空，并且将下标移动

index = startIndex - 1

state = 0

elif c == '*':

pass

else:

state = 2

def getMyProm():

'''从文件中获取代码片段'''

global _content

myPro = open(r'E://test.txt','r')

for line in myPro:

if line != '\n':

_content = "%s%s" %(_content,line.lstrip()) #效率更高的字符串拼接方法

else:

_content = "%s%s" %(_content,line)

myPro.close()

def analysis(mystr):

'''分析目标代码，生成token'''

global _p,_value,_syn,_mstate,_dstate,_line,_cstate

_value = ''

ch = mystr[_p]

_p += 1

while ch == ' ':

ch = mystr[_p]

_p += 1

if ch in string.letters or ch == '_': ###############letter(letter|digit)*

while ch in string.letters or ch in string.digits or ch == '_' or ch in _abnormalChar:

_value += ch

ch = mystr[_p]

_p += 1

_p -= 1

for abnormal in _abnormalChar:

if abnormal in _value:

_syn = '@-6' #错误代码，标识符中含有非法字符

break

else:

_syn = 'ID'

for s in _key:

if cmp(s,_value) == 0:

_syn = _value.upper() #############关键字

break

if _syn == 'ID':

inSymbolTable(_value)

elif ch == '\"': #############字符串

while ch in string.letters or ch in '\"% ' :

_value += ch

if _mstate == 0:

if ch == '\"':

_mstate = 1

elif _mstate == 1:

if ch == '\"':

_mstate = 2

ch = mystr[_p]

_p += 1

if _mstate == 1:

_syn = '@-2' #错误代码，字符串不封闭

_mstate = 0

elif _mstate == 2:

_mstate = 0

_syn = 'STRING'

_p -= 1

elif ch in string.digits:

while ch in string.digits or ch == '.' or ch in string.letters:

_value += ch

if _dstate == 0:

if ch == '0':

_dstate = 1

else:

_dstate = 2

elif _dstate == 1:

if ch == '.':

_dstate = 3

else:

_dstate = 5

elif _dstate == 2:

if ch == '.':

_dstate = 3

ch = mystr[_p]

_p += 1

for char in string.letters:

if char in _value:

_syn = '@-7' #错误代码，数字和字母混合，如12AB56等

_dstate = 0

if _syn != '@-7':

if _dstate == 5:

_syn = '@-3' #错误代码，数字以0开头

_dstate = 0

else:

_dstate = 0

if '.' not in _value:

_syn = 'DIGIT' ##################digit digit*

else:

if _value.count('.') == 1:

_syn = 'FRACTION' ################## 浮点数

else:

_syn = '@-5' #错误代码，浮点数中包含多个点，如1.2.3

_p -= 1

elif ch == '\'': ################## 字符

while ch in string.letters or ch in '@#$%&*\\\'\"':

_value += ch

if _cstate == 0:

if ch == '\'':

_cstate = 1

elif _cstate == 1:

if ch == '\\':

_cstate = 2

elif ch in string.letters or ch in '@#$%&*':

_cstate = 3

elif _cstate == 2:

if ch in 'nt':

_cstate = 3

elif _cstate == 3:

if ch == '\'':

_cstate = 4

ch = mystr[_p]

_p += 1

_p -= 1

if _cstate == 4:

_syn = 'CHARACTER'

_cstate = 0

else:

_syn = '@-4' #错误代码，字符不封闭

_cstate = 0

elif ch == '

_value = ch

ch = mystr[_p]

if ch == '=': ########### '<='

_value += ch

_p += 1

_syn = '<='

else: ########### '

_syn = '

elif ch == '>':

_value = ch

ch = mystr[_p]

if ch == '=': ########### '>='

_value += ch

_p += 1

_syn = '>='

else: ########## '>'

_syn = '>'

elif ch == '!':

_value = ch

ch = mystr[_p]

if ch == '=': ########## '!='

_value += ch

_p += 1

_syn = '!='

else: ########## '!'

_syn = '!'

elif ch == '+':

_value = ch

ch = mystr[_p]

if ch =='+': ############ '++'

_value += ch

_p += 1

_syn = '++'

else : ############ '+'

_syn = '+'

elif ch == '-':

_value = ch

ch = mystr[_p]

if ch =='-': ########### '--'

_value += ch

_p += 1

_syn = '--'

else : ########### '-'

_syn = '-'

elif ch == '=':

_value = ch

ch = mystr[_p]

if ch =='=': ########### '=='

_value += ch

_p += 1

_syn = '=='

else : ########### '='

_syn = '='

elif ch == '&':

_value = ch

ch = mystr[_p]

if ch == '&': ########### '&&'

_value += ch

_p += 1

_syn = '&&'

else: ########### '&'

_syn = '&'

elif ch == '|':

_value = ch

ch = mystr[_p]

if ch == '|': ########## '||'

_value += ch

_p += 1

_syn = '||'

else: ########## '|'

_syn = '|'

elif ch == '*': ########## '*'

_value = ch

_syn = '*'

elif ch == '/': ########## '/'

_value = ch

_syn = '/'

elif ch ==';': ########## ';'

_value = ch

_syn = ';'

elif ch == '(': ########## '('

_value = ch

_syn = '('

elif ch == ')': ########### ')'

_value = ch

_syn = ')'

elif ch == '{': ########### '{'

_value = ch

_syn = '{'

elif ch == '}': ########### '}'

_value = ch

_syn = '}'

elif ch == '[': ########### '['

_value = ch

_syn = '['

elif ch == ']': ########### ']'

_value = ch

_syn = ']'

elif ch == ',': ########## ','

_value = ch

_syn = ','

elif ch == '\n':

_syn = '@-1'

def inSymbolTable(token):

'''将关键字和标识符存进符号表'''

global _mysymbol

if token not in _mysymbol:

_mysymbol.append(token)

if __name__ == '__main__':

getMyProm()

outOfComment()

symbolTableFile = open(r'E://symbol_table.txt','w')

tokenFile = open(r'E://token.txt','w')

while _p != len(_content):

analysis(_content)

if _syn == '@-1':

_line += 1 #记录程序的行数

elif _syn == '@-2':

print '字符串 ' + _value + ' 不封闭! Error in line ' + str(_line)

elif _syn == '@-3':

print '数字 ' + _value + ' 错误，不能以0开头! Error in line ' + str(_line)

elif _syn == '@-4':

print '字符 ' + _value + ' 不封闭! Error in line ' + str(_line)

elif _syn == '@-5':

print '数字 ' + _value + ' 不合法! Error in line ' + str(_line)

elif _syn == '@-6':

print '标识符' + _value + ' 不能包含非法字符!Error in line ' + str(_line)

elif _syn == '@-7':

print '数字 ' + _value + ' 不合法,包含字母! Error in line ' + str(_line)

else: #若程序中无词法错误的情况

#print (_syn,_value)

tokenFile.write(str(_syn)+'@'+_value+'\n')

tokenFile.close()

symbolTableFile.write('入口地址\t变量名\n')

i = 0

for symbolItem in _mysymbol:

symbolTableFile.write(str(i)+'\t\t\t'+symbolItem+'\n')

i += 1

symbolTableFile.close()