【COMPILER 词法分析器 代码清单】

1.scannerclass.py

from enum import Enum
import math

#记号种类
Token_Type = Enum('Token_Type',
                                (#单词
                                'ORIGIN', 'SCALE', 'ROT', 'IS', 'TO', 'STEP', 'DRAW', 'FOR', 'FROM',
                                # 参数
                                'T',
                                # 分隔符
                                 'SEMICO', 'L_BRACKET','R_BRACKET','COMMA','Jing',
                                #运算符
                                 'PLUS','MINUS','MUL','DIV','POWER',
                                # 函数符
                                 'FUNC',
                                # 常数
                                 'CONST_ID',
                                #空记号
                                 'NONTOKEN',
                                # 出错记号
                                 'ERRTOKEN')
                  )


class Tokens:   #记号类
    #type：记号类别
    #lexeme：输入的字符串/属性
    #value：常数值
    #funcptr：函数指针
    def __init__(self,type,lexeme,value,funcptr):
        self.lexeme=lexeme
        self.value=value
        self.funcptr=funcptr
        if type in Token_Type:
            self.type = type
        else:
            print("Invalid type")     # 后续待填充

## 符号表
Alphabet=dict([('PI',Tokens(Token_Type.CONST_ID,"PI",3.1415926,None)),
           ('E',Tokens(Token_Type.CONST_ID,"E",2.71828,None)),		## 左key右value
           ('T',Tokens(Token_Type.T,'T',0.0,None)),
           ('SIN',Tokens(Token_Type.FUNC,'SIN',0.0,math.sin)),   # math.sin / math.sinh
           ('COS',Tokens(Token_Type.FUNC,'COS',0.0,math.cos)),
           ('TAN',Tokens(Token_Type.FUNC,'TAN',0.0,math.tan)),
           ('LN',Tokens(Token_Type.FUNC,'LN',0.0,math.log)),
           ('EXP',Tokens(Token_Type.FUNC,'EXP',0.0,math.exp)),
           ('SQRT',Tokens(Token_Type.FUNC,'SQRT',0.0,math.sqrt)),     # 后续操作待填充
           ('ORIGIN',Tokens(Token_Type.ORIGIN,'ORIGIN',0.0,None)),
           ('SCALE',Tokens(Token_Type.SCALE,'SCALE',0.0,None)),
           ('ROT',Tokens(Token_Type.ROT,'ROT',0.0,None)),
           ('IS',Tokens(Token_Type.IS,'IS',0.0,None)),
           ('FOR',Tokens(Token_Type.FOR,'FOR',0.0,None)),
           ('FROM',Tokens(Token_Type.FROM,'FROM',0.0,None)),
            ('TO',Tokens(Token_Type.TO,'TO',0.0,None)),
            ('STEP',Tokens(Token_Type.STEP, 'STEP', 0.0, None)),
            ('DRAW',Tokens(Token_Type.DRAW, 'DRAW', 0.0, None))])

2.scannerfunc.py

import scannerclass as sc
import os


class scanner():

    ##——————初始化词法分析器
    def __init__(self,file_name):   #输入要输入字符流的文件名
        self.LineNo = 0 #记录字符所在行的行号
        self.TokenBuffer = '' #待识别记号缓存区
        self.file_name = r''+file_name        #相对路径
        if os.path.exists(self.file_name):
            self.fp = open(self.file_name, "r",encoding='utf-8')     #文件指针
        else:
            self.fp = None

    ##——————关闭词法分析器
    def CloseScanner(self):
        if self.fp!=None:
            self.fp.close()

    ##——————从输入流中读入一个字符
    def GetChar(self):
        Char = self.fp.read(1)
        return Char

    ##——————输入流回退一个字符
    def BackChar(self,Char):        ## 非二进制打开方式不能直接seek目前位置回溯，所以用tell()-1方式从头跳转前一位置
        if Char != '':
            self.fp.seek(self.fp.tell()-1)

    ##——————加入字符到TokenBuffer待识别字符串中
    def AddCharToString(self,Char):
        self.TokenBuffer+=Char

    ##——————清空TokenBuffer字符串
    def EmptyString(self):
        self.TokenBuffer=''

    ##——————识别的字符串查表
    def JudgeKeyToken(self):
        Token=sc.Alphabet.get(self.TokenBuffer,sc.Tokens(sc.Token_Type.ERRTOKEN,self.TokenBuffer,0.0,None))
        return Token


    ##——————获取记号
    def GetToken(self):

        Char = ''   ##字符流
        type = ''   ##指向返回输出的Tokens对象
        self.EmptyString()  #清空缓冲区
        while(1):
            Char = self.GetChar()
            if Char == '':
                type = sc.Tokens(sc.Token_Type.NONTOKEN,Char,0.0,None)
                return type
            if Char == '\n':
                self.LineNo=self.LineNo+1
            if ~Char.isspace():
                break
        self.AddCharToString(Char)  ##若不是空格、TAB、回车、文件结束符等，则先加入到记号的字符缓冲区中
        if Char.isalpha():          ## 判断是否是英文
            while(1):
                Char = self.GetChar()
                if Char.isalnum():
                    self.AddCharToString(Char)
                else:
                    break
            self.BackChar(Char)
            type = self.JudgeKeyToken()
            type.lexeme = self.TokenBuffer
            return type

        elif Char.isdigit():
            while(1):
                Char = self.GetChar()
                if Char.isdigit():
                    self.AddCharToString(Char)
                else:
                    break
            if Char == '.':
                self.AddCharToString(Char)
                while(1):
                    Char = self.GetChar()
                    if Char.isdigit():
                        self.AddCharToString(Char)
                    else:
                        break
            self.BackChar(Char)
            type = sc.Tokens(sc.Token_Type.CONST_ID,self.TokenBuffer,int(self.TokenBuffer),None)
            return type

        else:
            if Char == ';':
                type = sc.Tokens(sc.Token_Type.SEMICO,Char,0.0,None)
            elif Char == '(':
                type = sc.Tokens(sc.Token_Type.L_BRACKET,Char,0.0,None)
            elif Char == ')':
                type = sc.Tokens(sc.Token_Type.R_BRACKET, Char, 0.0, None)
            elif Char == ',':
                type = sc.Tokens(sc.Token_Type.COMMA, Char, 0.0, None)
            elif Char == '+':
                type = sc.Tokens(sc.Token_Type.PLUS, Char, 0.0, None)
            elif Char == '-':   ##可能是行分割或减号
                Char = self.GetChar()
                if Char == '-':
                    while Char != '\n' and Char != '':
                        Char = self.GetChar()
                    self.BackChar(Char)
                    return self.GetToken()
                else:
                    self.BackChar(Char)
                    type = sc.Tokens(sc.Token_Type.MINUS, '-', 0.0, None)
            elif Char == '/':   ##可能是注释分割或除号
                Char = self.GetChar()
                if Char == '/':
                    while Char != '\n' and Char != '':
                        Char = self.GetChar()
                        self.BackChar(Char)
                        return self.GetToken()
                else:
                    self.BackChar(Char)
                    type = sc.Tokens(sc.Token_Type.DIV, '/', 0.0, None)
            # elif Char == '*':
            #     Char = self.GetChar()
            #     if (Char == '*'):
            #             type = sc.Tokens(sc.Token_Type.POWER, '**', 0.0, None)
            #     else:
            #         self.BackChar(Char)
            #         type = sc.Tokens(sc.Token_Type.MUL, '*', 0.0, None)
            elif Char == '*':
                type = sc.Tokens(sc.Token_Type.MUL, '*', 0.0, None)
            elif Char == '^':
                type = sc.Tokens(sc.Token_Type.POWER, '^', 0.0, None)

            elif Char == '#':   ##是注释分割
                Char = self.GetChar()
                while Char != '\n' and Char != '':
                    Char = self.GetChar()
                    self.BackChar(Char)
                    return self.GetToken()
                type = sc.Tokens(sc.Token_Type.Jing, '#', 0.0, None)
            else:
                type = sc.Tokens(sc.Token_Type.ERRTOKEN, Char, 0.0, None)
        return type

3.scannermain.py

import scannerclass as sc
import scannerfunc as sf

file_name = 'test^.txt'  ##测试文本相对地址
scanner = sf.scanner(file_name)

if scanner.fp != None:
    print('       记号类别              字符串      常数值     函数指针')
    #print('——————————————————————')
    while (1):
        token = scanner.GetToken()  # 输出一个记号
        if token.type == sc.Token_Type.Jing:  # '#'的注释
            continue
        if token.type == sc.Token_Type.ERRTOKEN:  # 忽略错误
            continue
            # 记号的类别不是错误或者空格，就打印出他的内容
        elif token.type != sc.Token_Type.NONTOKEN:
            print("{:20s} {:>12s} {:12f}  {}".format(token.type, token.lexeme, token.value, token.funcptr))
        else:
            break
else:
    print('Open Error!') #没找到测试文件

4.测试文件test.txt

FOR T FROM 0 TO 2*PI STEP PI/50 DRAW(COS(t),sin(t));