中文分词

qt在新版本中整合了qtlucene的组件，单位的机器qt版本不够，也懒得升级。先看一下中文切分词部分。

目的：对文档进行中文切分，并根据词频高低排序。

选择LibMMSeg 作为切分词组件。

下载源码，

./bootstrap

./configure --/prefix=/opt/mmseg

make

make install

完毕，

在qtcreator中新建项目，因为mmseg只提供了客户端开发文档，并未提及切分词部分的调用方法，暂时从 source code : src/mmseg_main.cpp 顺藤摸瓜。

以上代码为实现Demo:

项目文件：

[cpp] view plain copy print ?

1. QT += core webkit network
3. SOURCES += main.cpp \
4. mmseg_main.cpp
6. HEADERS +=
8. LIBS += -L/opt/mmseg/lib -lmmseg
10. INCLUDEPATH = /opt/mmseg/include/

QT += core webkit network

SOURCES += main.cpp \ 
    mmseg_main.cpp

HEADERS +=  

LIBS += -L/opt/mmseg/lib -lmmseg

INCLUDEPATH = /opt/mmseg/include/

main.cpp

[cpp] view plain copy print ?

1. #include <QApplication>
3. #include "mmseg_main.cpp"
5. int main(int argc, char *argv[])
6. {
7. QApplication app(argc, argv);
9. string dictpath = "/opt/mmseg/etc/";
10. string content = "全文检索是数据库的有力补充，全文检索并不能替代数据库在应用系统中的作用。当应用系统的数据以大量的文本信息为主时，采用全文检索技术可以极大的提升应用系统的价值。 ";
12. map<string, int> m = segment(content, dictpath);
14. vector<pair<string, int> > v;
15. sortMapByValue(m, v);
17. for (int i = 0; i < 50; ++i)
18. cout<<v[i].second<<"\t"<<v[i].first<<endl;
20. return 0;
22. // return app.exec();
23. }

#include <QApplication>

#include "mmseg_main.cpp"

int main(int argc, char *argv[])
{
    QApplication app(argc, argv);

    string dictpath = "/opt/mmseg/etc/";
    string content = "全文检索是数据库的有力补充，全文检索并不能替代数据库在应用系统中的作用。当应用系统的数据以大量的文本信息为主时，采用全文检索技术可以极大的提升应用系统的价值。 ";

    map<string, int> m = segment(content, dictpath);

    vector<pair<string, int> > v;
    sortMapByValue(m, v);

    for (int i = 0; i < 50; ++i)
          cout<<v[i].second<<"\t"<<v[i].first<<endl;

    return 0;

//    return app.exec();
}

mmseg_main.cpp

[cpp] view plain copy print ?

1. #include <fstream>
2. #include <string>
3. #include <iostream>
4. #include <cstdio>
6. #include <map>
7. #include <vector>
8. #include <algorithm>
10. #include <stdlib.h>
12. #include "mmseg/UnigramCorpusReader.h"
13. #include "mmseg/UnigramDict.h"
14. #include "mmseg/SynonymsDict.h"
15. #include "mmseg/ThesaurusDict.h"
16. #include "mmseg/SegmenterManager.h"
17. #include "mmseg/Segmenter.h"
19. using namespace std;
20. using namespace css;
22. #define SEGMENT_OUTPUT 1
24. map<string, int> segment(string content, string dictpath);
26. void addmap(const char* str, int len, map<string, int> &map);
28. void sortMapByValue(map<string,int>& tMap, vector<pair<string,int> >& tVector);
30. map<string, int> segment(string content, string dictpath)
31. {
32. map<string, int> map;
34. //init segmenter
35. const char* dict_path = dictpath.c_str();
36. SegmenterManager* mgr = new SegmenterManager();
37. mgr->init(dict_path);
38. Segmenter* seg = mgr->getSegmenter();
40. //load data.
41. int length = content.length();
42. const char* buffer = content.c_str();
44. //begin seg
45. seg->setBuffer((u1*)buffer,length);
46. u2 len = 0, symlen = 0;
47. u2 kwlen = 0, kwsymlen = 0;
49. //check 1st token.
50. char txtHead[3] = {239,187,191};
51. char* tok = (char*)seg->peekToken(len, symlen);
52. seg->popToken(len);
53. if(seg->isSentenceEnd()){
54. do {
55. char* kwtok = (char*)seg->peekToken(kwlen , kwsymlen,1);
56. if(kwsymlen)
57. // printf("[kw]%*.*s/x ",kwsymlen,kwsymlen,kwtok);
58. addmap(kwtok, kwsymlen, map);
59. }while(kwsymlen);
60. }
62. if(len == 3 && memcmp(tok,txtHead,sizeof(char)*3) == 0){
63. //check is 0xFEFF
64. //do nothing
65. }else{
66. // printf("%*.*s/x ",symlen,symlen,tok);
67. addmap(tok, symlen, map);
68. }
70. while(1){
71. len = 0;
72. char* tok = (char*)seg->peekToken(len,symlen);
73. if(!tok || !*tok || !len)
74. break;
75. seg->popToken(len);
76. if(seg->isSentenceEnd()){
77. do {
78. char* kwtok = (char*)seg->peekToken(kwlen , kwsymlen,1);
79. if(kwsymlen)
80. // printf("[kw]%*.*s/x ",kwsymlen,kwsymlen,kwtok);
81. addmap(kwtok, kwsymlen, map);
82. }while(kwsymlen);
83. }
85. if(*tok == '\r')
86. continue;
87. if(*tok == '\n'){
88. printf("\n");
89. continue;
90. }
92. //printf("[%d]%*.*s/x ",len,len,len,tok);
93. // printf("%*.*s/x ",symlen,symlen,tok);
95. // char word[20];
96. // sprintf(word, "%*.*s",symlen,symlen,tok);
97. // printf("%s/[%d-%d] ", word,symlen,symlen);
98. addmap(tok, symlen, map);
100. //check thesaurus
101. {
102. const char* thesaurus_ptr = seg->thesaurus(tok, symlen);
103. while(thesaurus_ptr && *thesaurus_ptr) {
104. len = strlen(thesaurus_ptr);
105. // printf("%*.*s/s ",len,len,thesaurus_ptr);
106. addmap(thesaurus_ptr, len, map);
107. thesaurus_ptr += len + 1; //move next
108. }
109. }
110. //printf("%s",tok);
111. }
113. delete mgr;
115. return map;
117. }
119. void addmap(const char* str, int len, map<string, int> &map)
120. {
121. char word[len];
122. sprintf(word, "%*.*s", len, len, str);
123. map[word]++;
124. }
126. int cmp(const pair<string,int>& a, const pair<string,int>& b)
127. {
128. return a.second > b.second;
129. }
131. void sortMapByValue(map<string,int>& tMap, vector<pair<string,int> >& tVector)
132. {
133. for(map<string,int>::iterator curr = tMap.begin();curr != tMap.end(); curr++)
134. {
135. tVector.push_back(make_pair(curr->first, curr->second));
136. }
138. sort(tVector.begin(),tVector.end(),cmp);
140. }

#include <fstream>
#include <string>
#include <iostream>
#include <cstdio>

#include <map>
#include <vector>
#include <algorithm>

#include <stdlib.h>

#include "mmseg/UnigramCorpusReader.h"
#include "mmseg/UnigramDict.h"
#include "mmseg/SynonymsDict.h"
#include "mmseg/ThesaurusDict.h"
#include "mmseg/SegmenterManager.h"
#include "mmseg/Segmenter.h"

using namespace std;
using namespace css;

#define SEGMENT_OUTPUT 1

map<string, int> segment(string content, string dictpath);

void addmap(const char* str, int len, map<string, int> &map);

void sortMapByValue(map<string,int>& tMap, vector<pair<string,int> >& tVector);

map<string, int> segment(string content, string dictpath)
{
    map<string, int> map;

    //init segmenter
    const char* dict_path = dictpath.c_str();
    SegmenterManager* mgr = new SegmenterManager();
    mgr->init(dict_path);
    Segmenter* seg = mgr->getSegmenter();

    //load data.
    int length = content.length();
    const char* buffer = content.c_str();

    //begin seg
    seg->setBuffer((u1*)buffer,length);
    u2 len = 0, symlen = 0;
    u2 kwlen = 0, kwsymlen = 0;

    //check 1st token.
    char txtHead[3] = {239,187,191};
    char* tok = (char*)seg->peekToken(len, symlen);
    seg->popToken(len);
    if(seg->isSentenceEnd()){
            do {
                    char* kwtok = (char*)seg->peekToken(kwlen , kwsymlen,1);
                    if(kwsymlen)
//                            printf("[kw]%*.*s/x ",kwsymlen,kwsymlen,kwtok);
                        addmap(kwtok, kwsymlen, map);
            }while(kwsymlen);
    }

    if(len == 3 && memcmp(tok,txtHead,sizeof(char)*3) == 0){
            //check is 0xFEFF
            //do nothing
    }else{
//            printf("%*.*s/x ",symlen,symlen,tok);
            addmap(tok, symlen, map);
    }

    while(1){
            len = 0;
            char* tok = (char*)seg->peekToken(len,symlen);
            if(!tok || !*tok || !len)
                    break;
            seg->popToken(len);
            if(seg->isSentenceEnd()){
                    do {
                            char* kwtok = (char*)seg->peekToken(kwlen , kwsymlen,1);
                            if(kwsymlen)
//                                    printf("[kw]%*.*s/x ",kwsymlen,kwsymlen,kwtok);
                                addmap(kwtok, kwsymlen, map);
                    }while(kwsymlen);
            }

            if(*tok == '\r')
                    continue;
            if(*tok == '\n'){
                    printf("\n");
                    continue;
            }

            //printf("[%d]%*.*s/x ",len,len,len,tok);
//                printf("%*.*s/x ",symlen,symlen,tok);

//            char word[20];
//            sprintf(word, "%*.*s",symlen,symlen,tok);
//            printf("%s/[%d-%d] ", word,symlen,symlen);
            addmap(tok, symlen, map);

            //check thesaurus
            {
                    const char* thesaurus_ptr = seg->thesaurus(tok, symlen);
                    while(thesaurus_ptr && *thesaurus_ptr) {
                            len = strlen(thesaurus_ptr);
//                            printf("%*.*s/s ",len,len,thesaurus_ptr);
                            addmap(thesaurus_ptr, len, map);
                            thesaurus_ptr += len + 1; //move next
                    }
            }
            //printf("%s",tok);
    }

    delete mgr;

    return map;

}

void addmap(const char* str, int len, map<string, int> &map)
{
    char word[len];
    sprintf(word, "%*.*s", len, len, str);
    map[word]++;
}

int cmp(const pair<string,int>& a, const pair<string,int>& b)
{
    return a.second > b.second;
}

void sortMapByValue(map<string,int>& tMap, vector<pair<string,int> >& tVector)
{
     for(map<string,int>::iterator curr = tMap.begin();curr != tMap.end(); curr++)
     {
        tVector.push_back(make_pair(curr->first, curr->second));
     }

     sort(tVector.begin(),tVector.end(),cmp);

}

输出结果：

[cpp] view plain copy print ?

1. 6 的
2. 3 应用
3. 3 检索
4. 3 全文
5. 3 系统
6. 2 数据库
7. 2 ，
8. 2 。
9. 1 能
10. 1 补充
11. 1 采用
12. 1 极大
13. 1 有力
14. 1 替代
15. 1 是
16. 1 时
17. 1 文本
18. 1 提升
19. 1 数据
20. 1
21. 1 技术
22. 1 当
23. 1 并不
24. 1 大量
25. 1 在
26. 1 可以
27. 1 信息
28. 1 作用
29. 1 价值
30. 1 以
31. 1 为主
32. 1 中