词法分析器

**

实验一 源程序的预处理及词法分析程序设计

**
**

一、 实验目的

设计并实现一个包含预处理功能的词法分析程序,加深对编译中词法分析过程的理解。

二、 实验要求

1、实现预处理功能
源程序中可能包含有对程序执行无意义的符号,要求将其剔除。
首先编制一个源程序的输入过程,从键盘、文件或文本框输入若干行语句,依次存入输入缓冲区(字符型数据);然后编制一个预处理子程序,去掉输入串中的回车符、换行符与制表符等编辑性文字;把多个空白符合并为一个;去掉注释。
2、实现词法分析功能
输入:所给文法的源程序字符串。
输出:二元组(syn,token或sum)构成的序列。其中,
syn为单词种别码。
Token为存放的单词自身字符串。
Sum为整型常量。
具体实现时,可以将单词的二元组用结构进行处理。

三、实现

1关键字表初值
关键字作为特殊标识符处理,把它们预先安排在一张表格中(称为关键字表),当扫描程序识别出标识符时,查关键字表。如能查到匹配的单词,则该单词为关键字,否则为一般标识符。关键字表为一个字符串数组,其描述如下:
char *rwtab[27]={“main”,”if”,”then”,”while”,”do”,” static”,”int”,” double”,”struct”,”break”,”else”,”long”,”switch”,”case”,”typedef”,”char”,”return”,”const”,”float”,”short”,”continue”,”for”,”void”,”default”,”sizeof”,”do”};

const char* rwtab[] = {
	"main",
	"if",
	"then",
	"while",
	"do",
	"static",
	"int",
	"double",
	"struct",
	"break",
	"else",
	"long",
	"switch",
	"case",
	"typedef",
	"char",
	"return",
	"const",
	"float",
	"short",
	"continue",
	"for",
	"void",
	"default",
	"sizeof",
	"do"
};

(2) 程序中需要用到的主要变量:syn,token和sum。
2. 扫描子程序的算法思想
设置三个变量:token用来存放构成单词符号的字符串;sum用来存放整型单词;syn用来存放单词符号的种别编码。读入文件,将文件中的数据存放在一个字符数组中,取名text.之后进行预处理preprocess。
3.预处理
预处理的任务是去掉输入串中的回车符、换行符与制表符等编辑性文字;把多个空白符合并为一个;去掉注释。
首先定义一个str字符数组用来存放处理过程中的数组。我们需要知道程序有效的部分从何处开始,也就是说我们必须跳过程序开头的注释。

    char str[1000];
    int p1 = 0;
    int p2 = 0;
    //去掉注释。
    if ((text[p1] == '/') && (text[p1 + 1] == '/'))
    {
        while (text[++p1] != '\n');
    }

这样p1所指向的字符就是第一个有效字符前的那一个换行符。
接下来我们开始清除掉程序有效部分开头多余的空格回车等。

ch = text[++p1];
    while ((ch == ' ')||( ch == '\n')||( ch == '\t'))
        ch = text[p1++];

接下来我们存入第一个字符(方便之后判断空格与处理)

 p1--;
    //存入第一个字符
    str[p2++] = text[p1++];

下面是一段循环来将有效字符存入str中,并且将编辑字符转化为一个空格。

while (text[p1] != '#')
    {
        //去掉注释。
        if ((text[p1] == '/') && (text[p1 + 1] == '/'))
        {
            while (text[++p1] != '\n');
        }
        //空格
        if ((text[p1] == ' ')|| (text[p1] == '\n')||(text[p1] == '\t'))
        {
            //保留且只保留一个空格
            //如果上一个字符不是空格,则这个字符为空格
            if (str[p2-1] != ' ')
                str[p2++] = ' ';
            p1++;
        }
        else
        {
            str[p2++] = text[p1++];
        }
    }

后续处理

    //加入#
    str[p2] = '#';
    for (p1 = 0; p1 < 1000; p1++)
        text[p1] = 0;
    for (p1 = 0; p1 <= p2; p1++)
        text[p1] = str[p1];

4.词法分析
接受一个字符,
ID→letter(letter|digit)*
NUM→digit digit*
letter→a|…|z|A|…|Z
digit→0|…|9…
由正则式我们可以很轻松的得到判断的逻辑。接受字符ch,如果是字符的话就说明将要识别出的那个东西是一个标识符或者自定义的字符串。

//标示符或者变量名 
    //ID→letter(letter|digit)*    字母开头
    if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'))
    {
        //读取这个字符串
        //并将这个字符串存入token中
        in_pos = 0;
        while ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'))
        {
            token[in_pos++] = ch;
            ch = text[pos++];
        }
        token[in_pos++] = '\0';
        pos--;
        syn = 25;//识别为自定义的字符
        for (int n = 0; n < 23; n++)  //将识别出来的字符和已定义的标示符作比较, 
        {
            if (strcmp(token, rwtab[n]) == 0)
            {
                syn = n + 1;
                break;
            }
        }
    }

数字同理

    //数字 
    else if ((ch >= '0' && ch <= '9'))
    {
        sum = 0;
        while ((ch >= '0' && ch <= '9'))
        {
            sum = sum * 10 + ch - '0';
            ch = text[pos++];
        }
        pos--;
        syn = 26;
    }//#

下面就剩下其他字符了。

    //其他字符
    else switch (ch)
    {
    case':':
        in_pos = 0;
        token[in_pos++] = ch;
        ch = text[pos++];
        if (ch == '=')//==
        {
            syn = 32;
            token[in_pos++] = ch;
        }
        else//=
        {
            syn = 38;
            pos--;
        }
        break;
    case'<':
        in_pos = 0;
        token[in_pos++] = ch;
        ch = text[pos++];
        if (ch == '>')//<>
        {
            syn = 34;
            token[in_pos++] = ch;
        }
        else if (ch == '=')//<=
        {
            syn = 35;
            token[in_pos++] = ch;
        }
        else//<
        {
            syn = 33;
            pos--;
        }
        token[in_pos++] = '\0';
        break;
    case'>':
        in_pos = 0;
        token[in_pos++] = ch;
        ch = text[pos++];
        if (ch == '=')//>=
        {
            syn = 37;
            token[in_pos++] = ch;
        }
        else//>
        {
            syn = 36;
            pos--;
        }
        token[in_pos++] = '\0';//#
        break;
    case'+':syn = 27; token[0] = ch; token[1] = '\0'; break;
    case'-':syn = 28; token[0] = ch; token[1] = '\0'; break;
    case'*':syn = 29; token[0] = ch; token[1] = '\0'; break;
    case'/':syn = 30; token[0] = ch; token[1] = '\0'; break;
    case'[':syn = 39; token[0] = ch; token[1] = '\0'; break;
    case']':syn = 40; token[0] = ch; token[1] = '\0'; break;
    case';':syn = 41; token[0] = ch; token[1] = '\0'; break;
    case'(':syn = 42; token[0] = ch; token[1] = '\0'; break;
    case')':syn = 43; token[0] = ch; token[1] = '\0'; break;
    case'#':syn = 0; token[0] = ch;  token[1] = '\0'; break;
    default:syn = -1;
    }

组合起来就得到了lexer函数

void lexer()
{
    ch = text[pos++];
    while (ch == ' ')
        ch = text[pos++];
    //标示符或者变量名 
    //ID→letter(letter|digit)*    字母开头
    if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'))
    {
        //读取这个字符串
        //并将这个字符串存入token中
        in_pos = 0;
        while ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'))
        {
            token[in_pos++] = ch;
            ch = text[pos++];
        }
        token[in_pos++] = '\0';
        pos--;
        syn = 25;//识别为自定义的字符
        for (int n = 0; n < 23; n++)  //将识别出来的字符和已定义的标示符作比较, 
        {
            if (strcmp(token, rwtab[n]) == 0)
            {
                syn = n + 1;
                break;
            }
        }
    }
    //数字 
    else if ((ch >= '0' && ch <= '9'))
    {
        sum = 0;
        while ((ch >= '0' && ch <= '9'))
        {
            sum = sum * 10 + ch - '0';
            ch = text[pos++];
        }
        pos--;
        syn = 26;
    }//#
    //其他字符
    else switch (ch)
    {
    case':':
        in_pos = 0;
        token[in_pos++] = ch;
        ch = text[pos++];
        if (ch == '=')//==
        {
            syn = 32;
            token[in_pos++] = ch;
        }
        else//=
        {
            syn = 38;
            pos--;
        }
        break;
    case'<':
        in_pos = 0;
        token[in_pos++] = ch;
        ch = text[pos++];
        if (ch == '>')//<>
        {
            syn = 34;
            token[in_pos++] = ch;
        }
        else if (ch == '=')//<=
        {
            syn = 35;
            token[in_pos++] = ch;
        }
        else//<
        {
            syn = 33;
            pos--;
        }
        token[in_pos++] = '\0';
        break;
    case'>':
        in_pos = 0;
        token[in_pos++] = ch;
        ch = text[pos++];
        if (ch == '=')//>=
        {
            syn = 37;
            token[in_pos++] = ch;
        }
        else//>
        {
            syn = 36;
            pos--;
        }
        token[in_pos++] = '\0';//#
        break;
    case'+':syn = 27; token[0] = ch; token[1] = '\0'; break;
    case'-':syn = 28; token[0] = ch; token[1] = '\0'; break;
    case'*':syn = 29; token[0] = ch; token[1] = '\0'; break;
    case'/':syn = 30; token[0] = ch; token[1] = '\0'; break;
    case'[':syn = 39; token[0] = ch; token[1] = '\0'; break;
    case']':syn = 40; token[0] = ch; token[1] = '\0'; break;
    case';':syn = 41; token[0] = ch; token[1] = '\0'; break;
    case'(':syn = 42; token[0] = ch; token[1] = '\0'; break;
    case')':syn = 43; token[0] = ch; token[1] = '\0'; break;
    case'#':syn = 0; token[0] = ch;  token[1] = '\0'; break;
    default:syn = -1;
    }
}

5.main函数

main函数只需要注意文件的读写操作。

int main()
{
    pos = 0;
    ifstream fin;
    ofstream fou;
    fin.open("example.txt");
    while (!fin.eof())
    {
        text[pos++] = fin.get();
    }
    fin.close();
    cout << text;
    pos = 0;
    preprocess();
    cout <<endl<<text;
    fou.open("out.txt");
    fou << text;
    fou.close();
    do
    {
        lexer();
        switch (syn)
        {
        case 25: cout << "(" << syn << "," << token << ")" << endl; break;
        case 26: cout << "(" << syn << "," << sum << ")" << endl; break;
        case -1: cout << "error" << endl; break;
        default:cout << "(" << syn << "," << token << ")" << endl; 
        }
    } while (syn != 0);
}

四、完整代码

main.cpp

#include <stdio.h>
#include <string.h>
#include <iostream>
#include <fstream>
using namespace std;
//预处理
void preprocess();
//词法分析
void lexer();

char text[1000], token[100];
char ch;
int syn;//种别码
int pos = 0;//位置
int in_pos = 0;//一个token中的位置
int sum = 0;
//keyword 0--24
const char* rwtab[] = {
    "main",
    "if",
    "then",
    "while",
    "do",
    "static",
    "int",
    "double",
    "struct",
    "break",
    "else",
    "long",
    "switch",
    "case",
    "typedef",
    "char",
    "return",
    "const",
    "float",
    "short",
    "continue",
    "for",
    "void",
    "default",
    "sizeof",
    "do"
};

int main()
{
    pos = 0;
    ifstream fin;
    ofstream fou;
    fin.open("example.txt");
    while (!fin.eof())
    {
        text[pos++] = fin.get();
    }
    fin.close();
    cout << text;
    pos = 0;
    preprocess();
    cout << endl << text;
    fou.open("out.txt");
    fou << text;
    fou.close();
    do
    {
        lexer();
        switch (syn)
        {
        case 25: cout << "(" << syn << "," << token << ")" << endl; break;
        case 26: cout << "(" << syn << "," << sum << ")" << endl; break;
        case -1: cout << "error" << endl; break;
        default:cout << "(" << syn << "," << token << ")" << endl;
        }
    } while (syn != 0);
}
void preprocess()
{
    char str[1000];
    int p1 = 0;
    int p2 = 0;
    //去掉注释。
    if ((text[p1] == '/') && (text[p1 + 1] == '/'))
    {
        while (text[++p1] != '\n');
    }

    //清除开头多余的空格
    ch = text[++p1];
    while ((ch == ' ') || (ch == '\n') || (ch == '\t'))
        ch = text[p1++];
    p1--;
    //存入第一个字符
    str[p2++] = text[p1++];
    while (text[p1] != '#')
    {
        //去掉注释。
        if ((text[p1] == '/') && (text[p1 + 1] == '/'))
        {
            while (text[++p1] != '\n');
        }
        //空格
        if ((text[p1] == ' ') || (text[p1] == '\n') || (text[p1] == '\t'))
        {
            //保留且只保留一个空格
            //如果上一个字符不是空格,则这个字符为空格
            if (str[p2 - 1] != ' ')
                str[p2++] = ' ';
            p1++;
        }
        else
        {
            str[p2++] = text[p1++];
        }
    }
    //加入#
    str[p2] = '#';
    for (p1 = 0; p1 < 1000; p1++)
        text[p1] = 0;
    for (p1 = 0; p1 <= p2; p1++)
        text[p1] = str[p1];
}
void lexer()
{
    ch = text[pos++];
    while (ch == ' ')
        ch = text[pos++];
    //标示符或者变量名 
    //ID→letter(letter|digit)*    字母开头
    if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'))
    {
        //读取这个字符串
        //并将这个字符串存入token中
        in_pos = 0;
        while ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'))
        {
            token[in_pos++] = ch;
            ch = text[pos++];
        }
        token[in_pos++] = '\0';
        pos--;
        syn = 25;//识别为自定义的字符
        for (int n = 0; n < 23; n++)  //将识别出来的字符和已定义的标示符作比较, 
        {
            if (strcmp(token, rwtab[n]) == 0)
            {
                syn = n + 1;
                break;
            }
        }
    }
    //数字 
    else if ((ch >= '0' && ch <= '9'))
    {
        sum = 0;
        while ((ch >= '0' && ch <= '9'))
        {
            sum = sum * 10 + ch - '0';
            ch = text[pos++];
        }
        pos--;
        syn = 26;
    }//#
    //其他字符
    else switch (ch)
    {
    case'=':
        in_pos = 0;
        token[in_pos++] = ch;
        ch = text[pos++];
        if (ch == '=')//==
        {
            syn = 32;
            token[in_pos++] = ch;
        }
        else//=
        {
            syn = 38;
            pos--;
        }
        token[in_pos++] = '\0';
        break;
    case'<':
        in_pos = 0;
        token[in_pos++] = ch;
        ch = text[pos++];
        if (ch == '>')//<>
        {
            syn = 34;
            token[in_pos++] = ch;
        }
        else if (ch == '=')//<=
        {
            syn = 35;
            token[in_pos++] = ch;
        }
        else if (ch == '<')//<<
        {
            syn = 52;
            token[in_pos++] = ch;
        }
        else//<
        {
            syn = 33;
            pos--;
        }
        token[in_pos++] = '\0';
        break;
    case'>':
        in_pos = 0;
        token[in_pos++] = ch;
        ch = text[pos++];
        if (ch == '=')//>=
        {
            syn = 37;
            token[in_pos++] = ch;
        }
        else if (ch == '>')//>>
        {
            syn = 53;
            token[in_pos++] = ch;
        }
        else//>
        {
            syn = 36;
            pos--;
        }
        token[in_pos++] = '\0';//#
        break;
    case'*':
        in_pos = 0;
        token[in_pos++] = ch;
        ch = text[pos++];
        if (ch == '*')//**
        {
            syn = 31;
            token[in_pos++] = ch;
        }
        else//*
        {
            syn = 29;
            pos--;
        }
        token[in_pos++] = '\0';//#
        break;
    case'+':syn = 27; token[0] = ch; token[1] = '\0'; break;
    case'-':syn = 28; token[0] = ch; token[1] = '\0'; break;
    case'/':syn = 30; token[0] = ch; token[1] = '\0'; break;
    case'[':syn = 39; token[0] = ch; token[1] = '\0'; break;
    case']':syn = 40; token[0] = ch; token[1] = '\0'; break;
    case';':syn = 41; token[0] = ch; token[1] = '\0'; break;
    case'(':syn = 42; token[0] = ch; token[1] = '\0'; break;
    case')':syn = 43; token[0] = ch; token[1] = '\0'; break;
    case'{':syn = 44; token[0] = ch; token[1] = '\0'; break;
    case'}':syn = 45; token[0] = ch; token[1] = '\0'; break;
    case'.':syn = 46; token[0] = ch; token[1] = '\0'; break;
    case'"':syn = 47; token[0] = ch; token[1] = '\0'; break;
    case'!':syn = 48; token[0] = ch; token[1] = '\0'; break;
    case':':syn = 49; token[0] = ch; token[1] = '\0'; break;
    case',':syn = 50; token[0] = ch; token[1] = '\0'; break;
    case'?':syn = 51; token[0] = ch; token[1] = '\0'; break;
    case'#':syn = 0; token[0] = ch;  token[1] = '\0'; break;
    default:syn = -1;
    }
}



五、结果

out.txt


int main() { pos = 0; ifstream fin; ofstream fou; fin.open("example.txt"); while (!fin.eof()) { text[pos++] = fin.get(); } fin.close(); cout << text; pos = 0; row = 1; pos = 0; preprocess(); fou.open("out.txt"); fou << text; fou.close(); do { lexer(); switch (syn) { case 25: cout << "(" << syn << "," << token << ")" << endl; break; case 26: cout << "(" << syn << "," << sum << ")" << endl; break; case -1: cout << "error" << endl; break; default:cout << "(" << syn << "," << token << ")" << endl; } } while (syn != 0); } #

在这里插入图片描述

©️2020 CSDN 皮肤主题: 大白 设计师:CSDN官方博客 返回首页