自制编程语言（第三弹）定义Token

终于到了激动人心的实现时候了。为了实现我们的自制语言，我们需要的步骤为：

1. 词法分析
2. 语法分析
3. 语义分析（此处不设置）
4. 解释器

详细完整的代码可以点击这里查看github项目。

---

词法分析：

将代码片段识别为关键词、标识符、操作符、数字字面量等Token。

首先我们定义需要定义我们语言中所用到所有token类型

public enum TokenType
{
    NAME,
    INT_CONST,
    REAL_CONST,
    STR,
    NEWLINE,
    INDENT,
    DEDENT,
    COLON,
    LPAREN,
    RPAREN,
    LBRACK,
    RBRACK,
    ASSIGN,
    COMMENT,
    DOT,
    IF,
    ELSE,
    ELIF,
    CASE,
    WHEN,
    EQ,
    NE,
    LT,
    LE,
    GT,
    GE,
    PLUS,
    MINUS,
    MUL,
    DIV,
    FLOORDIV,
    MOD,
    POW,
    NOT,
    AND,
    OR,
    TRUE,
    FALSE,
    NONE,
    EOF
}

定义一个Token类，用于封装和包含Token信息

public class Token

public TokenType Type { get; set; }
public object Value { get; set; }
public int Line { get; set; }
public int Column { get; set; }

对保留字做个映射

public static Dictionary<string, TokenType> ReservedKeywords { get; } = new Dictionary<string, TokenType>()
{
    {"的",TokenType.DOT },
    {"如果",TokenType.IF},
    {"否则",TokenType.ELSE},
    {"否则如果",TokenType.ELIF},
    {"选择",TokenType.CASE},
    {"选项",TokenType.WHEN},
    {"非",TokenType.NOT},
    {"并且",TokenType.AND},
    {"或者",TokenType.OR},
    {"幂",TokenType.POW},
    {"true",TokenType.TRUE},
    {"false",TokenType.FALSE},
    {"none",TokenType.NONE},
    {"真",TokenType.TRUE},
    {"假",TokenType.FALSE},
    {"开",TokenType.TRUE},
    {"关",TokenType.FALSE},
    {"空",TokenType.NONE},
};
public static Token GetReservedKeywords(string value, int line, int column)
{
    if (ReservedKeywords.ContainsKey(value))
        return new Token(ReservedKeywords[value], value, line, column);
    return null;
}

设置构造函数和ToString

public Token(TokenType type, object value, int line, int column)
{
    Type = type;
    Value = value;
    Line = line;
    Column = column;
}
public override string ToString()
{
    return string.Format("Token({0}, {1}, position = {2}:{3})", Type, Value, Line, Column);
}