Python如何制作一款编程语言 EP·3Lexer分析器

一·什么是Lexer
        词法分析（英语：lexical analysis）是计算机科学中将字符序列转换为标记（token）序列的过程。进行词法分析的程序或者函数叫作词法分析器（lexical analyzer，简称lexer），也叫扫描器（scanner）。词法分析器一般以函数的形式存在，供语法分析器调用。

                                                                                ——zh.wikipedia.org

二·编写Lexer

        一个重要的功能就是算数，我偷个懒就用了python的eval函数，但是eval函数可以用来识别python的语法，所以我就写了个小判断；如果有字符串类型包含在tokens就说明有问题（这就是为什么Token类有个类型叫variable用于区分字符串和变量）

        

class Lexer:
    def evaluate(self, expression):
        tokens = Token.tokens(''.join(expression))
        for t in tokens:
            if CHAR in t and not (
                    tokens[0] == DOUBLE and tokens[-1] == DOUBLE or tokens[0] == SINGLE and tokens[-1] == SINGLE):
                Error.ValueErrors(["IF", expression])
            else:
                return eval(''.join(expression))

        还有一个至关重要的功能：布尔值运算：

KEY = [
    "AND",
    "OR",
    "NOT"
]
class Lexer:
    def BOOL(self, expression) -> bool:
        tokens = Token.tokens(expression)
        e = Parser.cutstring(expression.replace("TRUE", "1").replace("FALSE", "0"), tokens).replace(" ", "")
        state = []
        isTrue = False
        for i in range(len(e)):
            if KEY[0] in e:
                e = e.replace(" ", "").split(KEY[0])
                for e_ in e:
                    for i in range(len(e_)):
                        if e_[i] == "=" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left == right:
                                state.append(True)
                            else:
                                state.append(False)
                        if e_[i] == ">" and e_[i + 1] != "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 1:])
                            if left > right:
                                state.append(True)
                            else:
                                state.append(False)
                        if e_[i] == ">" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left >= right:
                                state.append(True)
                            else:
                                state.append(False)
                        if e_[i] == "<" and e_[i + 1] != "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 1:])
                            if left < right:
                                state.append(True)
                            else:
                                state.append(False)
                        if e_[i] == "<" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left <= right:
                                state.append(True)
                            else:
                                state.append(False)
                        if e_[i] == "!" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left != right:
                                state.append(True)
                            else:
                                state.append(False)

                for i in range(1, len(state)):
                    if state[0] == state[i]:
                        isTrue = True
                    else:
                        isTrue = False
                return isTrue

            if KEY[1] in e:
                e = e.replace(" ", "").split(KEY[1])
                for e_ in e:
                    for i in range(len(e_)):
                        if e_[i] == "=" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left == right:
                                return True
                        if e_[i] == ">" and e_[i + 1] != "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 1:])
                            if left > right:
                                return True
                        if e_[i] == ">" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left >= right:
                                return True
                        if e_[i] == "<" and e_[i + 1] != "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 1:])
                            if left < right:
                                return True
                        if e_[i] == "<" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left <= right:
                                return True
                        if e_[i] == "!" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left != right:
                                return True
                return False

            if KEY[2] in e:
                e = e.replace(" ", "").split(KEY[2])
                for e_ in e:
                    for i in range(len(e_)):
                        if e_[i] == "=" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left == right:
                                return not True
                            else:
                                return not False
                        if e_[i] == ">" and e_[i + 1] != "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 1:])
                            if left > right:
                                return not True
                            else:
                                return not False
                        if e_[i] == ">" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left >= right:
                                return not True
                            else:
                                return not False
                        if e_[i] == "<" and e_[i + 1] != "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 1:])
                            if left < right:
                                return not True
                            else:
                                return not False
                        if e_[i] == "<" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left <= right:
                                return not True
                            else:
                                return not False
                        if e_[i] == "!" and e_[i + 1] == "=":
                            left = self.evaluate(e_[:i])
                            right = self.evaluate(e_[i + 2:])
                            if left != right:
                                return not True
                            else:
                                return not False

            if KEY[0] not in e and KEY[1] not in e and KEY[2] not in e:
                if e[i] == "=" and e[i+1] == "=":
                    left = self.evaluate(e[:i].replace(" ", ""))
                    right = self.evaluate(e[i+2:].replace(" ", ""))
                    if left == right:
                        return True
                    else:
                        return False
                if e[i] == ">" and e[i + 1] != "=":
                  left = self.evaluate(e[:i])
                  right = self.evaluate(e[i + 1:])
                  if left > right:
                      return True
                  else:
                      return False
                if e[i] == ">" and e[i + 1] == "=":
                  left = self.evaluate(e[:i])
                  right = self.evaluate(e[i + 2:])
                  if left >= right:
                      return True
                  else:
                      return False
                if e[i] == "<" and e[i + 1] != "=":
                  left = self.evaluate(e[:i])
                  right = self.evaluate(e[i + 1:])
                  if left < right:
                      return True
                  else:
                      return False
                if e[i] == "<" and e[i + 1] == "=":
                  left = self.evaluate(e[:i])
                  right = self.evaluate(e[i + 2:])
                  if left <= right:
                      return True
                  else:
                      return False
                if e[i] == "!" and e[i + 1] == "=":
                    left = self.evaluate(e[:i])
                    right = self.evaluate(e[i + 2:])
                    if left != right:
                        return True
                    else:
                        return False
        return False

三·Lexer完成
        文章结构总览

                EP0·引言

                EP1·Tokens辨析器

                EP2·Lexer分析器（<o）

                EP3·Parser解析器

                EP4·AST抽象语法树

                EP5·正式编写

                        1. 输出（PRINT）

                        2. 输入 (INPUT)

                        3. 变量 (VAR)

                        4. 循环（FOR）

                        5. 条件判断（IF）