NLP的文本分类过程中,大多会统计文章的词频,这是分类的重要依据之一。词频是由一个pair组成的,word是key
frequece是value。用什么方法统计最好,当然是map。用vector,list也可以实现,但是它们基于关键字的检索效率没有
map高,map一般是用rb-Tree实现的,查找效率是O(log(n)),list,vector都是线性的,查找复杂度是O(n)。
先上代码。
![](https://i-blog.csdnimg.cn/blog_migrate/8f900a89c6347c561fdf2122f13be562.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/961ddebeb323a10fe0623af514929fc1.gif)
#ifndef _WORD_FREQUENCE_
#define _WORD_FREQUENCE_
#include < map >
#include < iostream >
#include < string >
using std::map;
class WordFrequence{
public :
WordFrequence(): file_name_(NULL){}
WordFrequence( char * file_name): file_name_(file_name){
LoadFromFile();
ReplaceSymbol();
parse();
}
private :
char * file_name_;
char * text;
map < std:: string , int > word_frequence_map_;
void parse();
void ReplaceSymbol();
void LoadFromFile();
bool IsWhiteChar( const char chr);
friend std::ostream & operator << (std::ostream & os, const WordFrequence & wf);
};
#endif
#define _WORD_FREQUENCE_
#include < map >
#include < iostream >
#include < string >
using std::map;
class WordFrequence{
public :
WordFrequence(): file_name_(NULL){}
WordFrequence( char * file_name): file_name_(file_name){
LoadFromFile();
ReplaceSymbol();
parse();
}
private :
char * file_name_;
char * text;
map < std:: string , int > word_frequence_map_;
void parse();
void ReplaceSymbol();
void LoadFromFile();
bool IsWhiteChar( const char chr);
friend std::ostream & operator << (std::ostream & os, const WordFrequence & wf);
};
#endif
![](https://i-blog.csdnimg.cn/blog_migrate/8f900a89c6347c561fdf2122f13be562.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/961ddebeb323a10fe0623af514929fc1.gif)
#include
"
word_frequence.h
"
#include < string >
#include < iostream >
#include < fstream >
#include < map >
const char * symbols = " ~!@#$%^&*()_+-=[]\\{}|:\ " ; ' ,./<>?";
const int MAX_SIZE = 100000 ;
bool WordFrequence::IsWhiteChar( const char chr){
switch (chr){
case ' \t ' :
case ' \r ' :
case ' \n ' :
case ' ' :
case ' \0 ' :
return true ;
default :
return false ;
}
}
void WordFrequence::LoadFromFile(){
std::ifstream is (file_name_, std::fstream:: in );
if ( ! is )
std::cerr << " error: can't open file: " << " [ " << file_name_ << " ] " << std::endl;
text = new char [MAX_SIZE];
is .read(text, MAX_SIZE);
}
void WordFrequence::parse(){
word_frequence_map_.clear();
int index = 0 ;
int count = strlen(text);
std:: string str;
while (index < count){
for ( int i = index; i <= count; ++ i){
if (IsWhiteChar(text[i])){
int len = i - index + 1 ;
char * p = new char [len];
memcpy(p, text + index, i - index);
p[len - 1 ] = ' \0 ' ;
str = p;
++ word_frequence_map_[str];
index = i + 1 ;
while (IsWhiteChar(text[index]))
++ index;
break ;
}
}
}
}
void WordFrequence::ReplaceSymbol(){
int j = 0 ;
while ( * (text + j) != ' \0 ' ){
for ( int i = 0 ; i < strlen(symbols); ++ i){
if ( * (text + j) == symbols[i])
* (text + j) = ' ' ;
}
j ++ ;
}
}
std::ostream & operator << (std::ostream & os, const WordFrequence & wf){
os << " word\t\tfrequence " << std::endl;
os << " ----------------------- " << std::endl;
std::map < std:: string , int > ::const_iterator i_begin = wf.word_frequence_map_.begin();
std::map < std:: string , int > ::const_iterator i_end = wf.word_frequence_map_.end();
while (i_begin != i_end){
os << "" << i_begin -> first << " \t\t " << i_begin -> second << "" << std::endl;
++ i_begin;
}
return os;
}
#include < string >
#include < iostream >
#include < fstream >
#include < map >
const char * symbols = " ~!@#$%^&*()_+-=[]\\{}|:\ " ; ' ,./<>?";
const int MAX_SIZE = 100000 ;
bool WordFrequence::IsWhiteChar( const char chr){
switch (chr){
case ' \t ' :
case ' \r ' :
case ' \n ' :
case ' ' :
case ' \0 ' :
return true ;
default :
return false ;
}
}
void WordFrequence::LoadFromFile(){
std::ifstream is (file_name_, std::fstream:: in );
if ( ! is )
std::cerr << " error: can't open file: " << " [ " << file_name_ << " ] " << std::endl;
text = new char [MAX_SIZE];
is .read(text, MAX_SIZE);
}
void WordFrequence::parse(){
word_frequence_map_.clear();
int index = 0 ;
int count = strlen(text);
std:: string str;
while (index < count){
for ( int i = index; i <= count; ++ i){
if (IsWhiteChar(text[i])){
int len = i - index + 1 ;
char * p = new char [len];
memcpy(p, text + index, i - index);
p[len - 1 ] = ' \0 ' ;
str = p;
++ word_frequence_map_[str];
index = i + 1 ;
while (IsWhiteChar(text[index]))
++ index;
break ;
}
}
}
}
void WordFrequence::ReplaceSymbol(){
int j = 0 ;
while ( * (text + j) != ' \0 ' ){
for ( int i = 0 ; i < strlen(symbols); ++ i){
if ( * (text + j) == symbols[i])
* (text + j) = ' ' ;
}
j ++ ;
}
}
std::ostream & operator << (std::ostream & os, const WordFrequence & wf){
os << " word\t\tfrequence " << std::endl;
os << " ----------------------- " << std::endl;
std::map < std:: string , int > ::const_iterator i_begin = wf.word_frequence_map_.begin();
std::map < std:: string , int > ::const_iterator i_end = wf.word_frequence_map_.end();
while (i_begin != i_end){
os << "" << i_begin -> first << " \t\t " << i_begin -> second << "" << std::endl;
++ i_begin;
}
return os;
}
#include
<
iostream
>
#include " word_frequence.h "
using namespace std;
int main( int argc, char * argv[])
{
WordFrequence wf( " d:\\test.txt " );
return 0 ;
}
#include " word_frequence.h "
using namespace std;
int main( int argc, char * argv[])
{
WordFrequence wf( " d:\\test.txt " );
return 0 ;
}
实现的方式很简单,首先把从文件里load出text,然后去掉里面的符号,最后扫描一遍文件,遇着单词就塞到map
里面.
++word_freq_map["word"];
这句话太好用了。一句话实现插入map,如果有就增加value,如果没有就插入。
这个程序简单训练了一下map容器的使用方法,也用到文件的读取。注意ostream open以后一定要判断open
成功了没有。ostream有几种读取方式,有格式化的>>读取,也有getline这种一行读取的,也有getchar这种一个字符
读一次的。也有read这种一次读一大段二进制的。读的时候一定要对文件内容有先验知识。
如果一次读的数据量很大,建议read来读取,效率很高,用循环读取可能效率很低。