N-gram

#include "iostream"
#include "string.h"
#include "string"
#include "map"
#include "fstream"
#include "set"
#include "vector"
#include "stdio.h"
using namespace std;

vector<string> words;          //词汇库
set<string> stopWords;         //停用词
map<string, float> dictMakeUni;  //制作一元词典时用到
map<string, float> dictMakeBi;   //制作二元词典时用到
map<string, float> dictMakeTri;  //制作三元词典时用到
map<string, float> dictReadUni;  //读取一元词典时用到
map<string, float> dictReadBi;   //读取二元词典时用到
map<string, float> dictReadTri;  //读取三元词典时用到
vector<string> wordList;       //将测试语句分词后的结果
const int maxLength = 20;      //最长单词：10，一个汉字的长度是2
const int laplace = 1;  //平滑值
string stringBi;     //二元分词结果
string stringTri;    //三元分词结果
double bestRate = 0;  //句子切分概率
float smallP;        //一个小概率值
int allNum = 0;      //所有词的总数

//读取停用词，这里只是一些标点符号
void readStopWords(char *file)
{
	stopWords.clear();
	ifstream fin;
	fin.open(file);
	if(!fin)
		cout << "停用词文件不存在";
	char buf[20];
	while(!fin.eof())
	{
		fin.getline(buf, 20);
		stopWords.insert(string(buf));
	}
	fin.close();
}

//制作词汇表、一元词典
void makeWords(char *file)
{
	words.clear();
	ifstream fin;
	fin.open(file);
	if(!fin)
		cout << "语料库不存在";
	dictMakeUni.clear();
	ofstream foutKey, foutVal;
	foutKey.open("dictKeyUni.txt");
	if(!foutKey)
		cout << "dictKeyUni.txt创建失败";
	foutVal.open("dictValUni.txt");
	if(!foutVal)
		cout << "dictValUni.txt创建失败";
	
	char buf[500];
	char *d = " ";
	float total = 0;
	while(!fin.eof())
	{
		fin.getline(buf, 500);
		char *p;
		p = strtok(buf, d);
		while(p)
		{
			string word(p);
			if(stopWords.find(word) == stopWords.end())  //不属于停用词
			{
				if(dictMakeUni.find(word) != dictMakeUni.end())
					dictMakeUni[word] += 1;
				else
					dictMakeUni[word] = 1;  //至少出现一次
				words.push_back(word);
				total++;
			}
			p = strtok(NULL, d);
		}
	}
	
	allNum = total;
	smallP = 1.0/(float)total;
	map<string, float>::iterator it;
	for(it = dictMakeUni.begin(); it != dictMakeUni.end(); it++)
	{
		foutKey << it->first << "\n";
		foutVal << (float)it->second/total << "\n";
	}
	
	foutKey.close();
	foutVal.close();
}

//制作二元词典
void makeDictBi()
{
	dictMakeBi.clear();
	ofstream foutKey, foutVal;
	foutKey.open("dictKeyBi.txt");
	if(!foutKey)
		cout << "dictKeyBi.txt创建失败";
	foutVal.open("dictValBi.txt");
	if(!foutVal)
		cout << "dictValBi.txt创建失败";
	
	float total = 0;
	for(int i=0; i<words.size()-1; i++)
	{
		string word = words[i] + words[i+1];
		total++;
		if(dictMakeBi.find(word) != dictMakeBi.end())
			dictMakeBi[word] += 1; 
		else
			dictMakeBi[word] = 1;  //至少出现一次
	}
	
	map<string, float>::iterator it;
	for(it = dictMakeBi.begin(); it != dictMakeBi.end(); it++)
	{
		foutKey << it->first << "\n";
		foutVal << (float)it->second/total << "\n";
	}
	
	foutKey.close();
	foutVal.close();
}

//制作三元词典
void makeDictTri()
{
	dictMakeTri.clear();
	ofstream foutKey, foutVal;
	foutKey.open("dictKeyTri.txt");
	if(!foutKey)
		cout << "dictKeyTri.txt创建失败";
	foutVal.open("dictValTri.txt");
	if(!foutVal)
		cout << "dictValTri.txt创建失败";
	
	float total = 0;
	for(int i=0; i<words.size()-2; i++)
	{
		string word = words[i] + words[i+1] + words[i+2];
		total++;
		if(dictMakeTri.find(word) != dictMakeTri.end())
			dictMakeTri[word] += 1; 
		else
			dictMakeTri[word] = 1;  //至少出现一次
	}
	
	map<string, float>::iterator it;
	for(it = dictMakeTri.begin(); it != dictMakeTri.end(); it++)
	{
		foutKey << it->first << "\n";
		foutVal << (float)it->second/total << "\n";
	}
	
	foutKey.close();
	foutVal.close();
}

//读取makeDictUni()中制作好的词典，一元词典
void readDictUni()
{
	dictReadBi.clear();
	ifstream finKey, finVal;
	finKey.open("dictKeyUni.txt");
	if(!finKey)
		cout << "dictKeyUni.txt不存在";
	finVal.open("dictValUni.txt");
	if(!finVal)
		cout << "dictValUni.txt不存在";
	
	char buf1[50], buf2[50];
	char *d = " ";
	while(!finKey.eof() && !finVal.eof())
	{
		finKey.getline(buf1, 50);
		finVal.getline(buf2, 50);
		dictReadUni[string(buf1)] = atof(buf2);
	}
	
	finKey.close();
	finVal.close();
}

//读取makeDictBi()中制作好的词典，二元词典
void readDictBi()
{
	dictReadBi.clear();
	ifstream finKey, finVal;
	finKey.open("dictKeyBi.txt");
	if(!finKey)
		cout << "dictKeyBi.txt不存在";
	finVal.open("dictValBi.txt");
	if(!finVal)
		cout << "dictValBi.txt不存在";
	
	char buf1[50], buf2[50];
	char *d = " ";
	while(!finKey.eof() && !finVal.eof())
	{
		finKey.getline(buf1, 50);
		finVal.getline(buf2, 50);
		dictReadBi[string(buf1)] = atof(buf2);
	}
	
	finKey.close();
	finVal.close();
}

//读取makeDictTri()中制作好的词典，三元词典
void readDictTri()
{
	dictReadTri.clear();
	ifstream finKey, finVal;
	finKey.open("dictKeyTri.txt");
	if(!finKey)
		cout << "dictKeyTri.txt不存在";
	finVal.open("dictValTri.txt");
	if(!finVal)
		cout << "dictValTri.txt不存在";
	
	char buf1[50], buf2[50];
	char *d = " ";
	while(!finKey.eof() && !finVal.eof())
	{
		finKey.getline(buf1, 50);
		finVal.getline(buf2, 50);
		dictReadTri[string(buf1)] = atof(buf2);
	}
	
	finKey.close();
	finVal.close();
}

void biGram(string &sentence, int len, int end, double rate, int index, vector<string> &wordList)
{
	if(len <= 2)   //句子只有一个字
	{
		stringBi = sentence;
		return;
	}
	if(end >= len-2)  //句子已经划分完
	{
		string str = "";
		for(int i=0; i<wordList.size(); i++)
			str += wordList[i] + " ";
		cout << str << "\t" << "概率：" << rate << endl;
		if(rate >= bestRate)   //当前的划分方式更好
		{
			
			stringBi = str;
			bestRate = rate;
		}
		return;
	}
	if(end == -1)  //第一次执行，需做清空工作
	{
		wordList.clear();
		stringBi = "";
		bestRate = 0;
	}

	string word = "";
	int last = end + 1;
	for(int i=end+1; i<len && i<maxLength; i+=2)
	{
		word += sentence.substr(i, 2);
		if(index == 0) // 如果是第一个词
		{
			if(i == last || dictReadUni.find(word) != dictReadUni.end())
			{
				wordList.push_back(word);
				biGram(sentence, len, i+1, rate, index+1, wordList);
				wordList.pop_back();
			}
		}
		else //不是第一个词
		{
			string preWord = wordList[index-1];
			string twoWords = preWord + word;
			if(i == last  || dictReadUni.find(word) != dictReadUni.end())  //如果是第一个字，或者存在于一元词典中
			{
				if(dictReadBi.find(twoWords) != dictReadBi.end())  //如果存在2-gram
				{
					//求条件概率P(w_i | w_i-1) = P(w_i-1 w_i) / P(w_i-1)
					if(dictReadUni.find(preWord) != dictReadUni.end())    //前面的词存在于一元词典中
					{
						wordList.push_back(word);
						biGram(sentence, len, i + 1, rate * (dictReadBi[twoWords] / dictReadUni[preWord]), index + 1, wordList);
						wordList.pop_back();
					}
					else  //前面的词不存在于一元词典中
					{
						wordList.push_back(word);
						biGram(sentence, len, i + 1, rate * dictReadBi[twoWords], index + 1, wordList);
						wordList.pop_back();
					}
				}
				else  //不存在2-gram
				{
					wordList.push_back(word);
					biGram(sentence, len, i + 1, rate * smallP, index + 1, wordList);
					wordList.pop_back();
				}
			}
		}
	}
}

//返回句子分词后的结果，三元模型
void triGram(string &sentence, int len, int end, double rate, int index, vector<string> &wordList)
{
	if(len <= 2)  //句子只有1个字
	{
		stringTri = sentence;
		return;
	}
	if(end >= len-2)  //句子已经划分完
	{
		string str = "";
		for(int i=0; i<wordList.size(); i++)
			str += wordList[i] + " ";
		cout << str << "\t" << "概率：" << rate << endl;
		if(rate >= bestRate)  //当前的划分方式更好
		{
			stringTri = str;
			bestRate = rate;
		}
		return;
	}
	if(end == -1)  //第一次执行本程序，需做清空工作
	{
		wordList.clear();
		stringTri = "";
		bestRate = 0;
	}

	string word = "";
	int last = end + 1;
	for(int i=end+1; i<len && i<maxLength; i+=2)
	{
		word += sentence.substr(i, 2);
		if(index == 0 || index ==1)  //第一个词或者第二个词
		{
			if(i == last || dictReadUni.find(word) != dictReadUni.end())  
			{
				wordList.push_back(word);
				triGram(sentence, len, i+1, rate * 0.9, index+1, wordList);
				wordList.pop_back();
			}
		}
		else
		{
			string preWords = wordList[index-2] + wordList[index-1];
			string threeWords = preWords + word;
			if(i == last || dictReadUni.find(word) != dictReadUni.end())  //第一个字，或者存在于一元词典中
			{
				//要计算P(w_i | w_i-2 w_i-1)
				//P(w_i | w_i-2 w_i-1) = P(w_i-2 w_i-1 w_i) / P(w_i-1 w_i)
				if(dictReadTri.find(threeWords) != dictReadTri.end())  //存在3-gram
				{
					if(dictReadBi.find(preWords) != dictReadBi.end())  //前2个词存在于二元词典中
					{
						wordList.push_back(word);
						triGram(sentence, len, i + 1, rate * (dictReadTri[threeWords] / dictReadBi[preWords]), index + 1, wordList);
						wordList.pop_back();
					}
					else
					{
						wordList.push_back(word);
						triGram(sentence, len, i + 1, rate * dictReadTri[threeWords], index + 1, wordList);
						wordList.pop_back();
					}
				}
				else   //不存在3-gram
				{
					wordList.push_back(word);
					triGram(sentence, len, i + 1, rate * smallP, index + 1, wordList);
					wordList.pop_back();
				}
			}
		}
	}
}

int main()
{
	//只在程序第一次启用的时候需要
	readStopWords("stopwords.txt");   //读取停用词
	makeWords("train.txt");  //读取语料库，制作一元词典
	makeDictBi();      //制作二元词典
	makeDictTri();     //制作三元词典
	///
	
	readDictUni();     //读取制作好的一元词典
	readDictBi();      //读取制作好的二元词典
	readDictTri();     //读取制作好的三元词典

	string str;
	cout << "请输入句子：\t";
	while(cin >> str)
	{
		int end = -1;
		int index = 0;
		double rate = 1;
		int len = str.size();
		cout << "\n二元模型分词过程：\n";
		biGram(str, len, end, rate, index, wordList);  //二元模型划分句子
		cout << "二元模型切分结果：\t" << stringBi << endl << endl;

		end = -1;
		index = 0;
		rate = 1;
		cout << "三元模型分词过程：\n";
		triGram(str, len, end, rate, index, wordList);  //三元模型划分句子
		cout << "三元模型切分结果：\t" << stringTri << endl << endl;
		cout << endl << endl;
		cout << "请输入句子：\t";
	}
	return 0;
}