在实现了中文切词的基础方法上,我将其封装在继承lucene的Analyzer类下
chineseAnalzer的方法就不用多说了。
using
System;
using
System.Collections.Generic;
using
System.Text;
![](https://i-blog.csdnimg.cn/blog_migrate/f0cd6c7f9e7ae96feae062cb48f670f0.gif)
using
Lucene.Net.Analysis;
using
Lucene.Net.Analysis.Standard;
![](https://i-blog.csdnimg.cn/blog_migrate/f0cd6c7f9e7ae96feae062cb48f670f0.gif)
namespace
Lucene.Fanswo
![](https://i-blog.csdnimg.cn/blog_migrate/34031c708bfe702fe82d01ff5c6593aa.gif)
{
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
/**//// <summary>
///
/// </summary>
public class ChineseAnalyzer:Analyzer
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
//private System.Collections.Hashtable stopSet;
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
public static readonly System.String[] CHINESE_ENGLISH_STOP_WORDS = new System.String[]
{ "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "no", "not", "of", "on", "or", "s", "such", "t", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with", "我", "我们" };
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
/**//// <summary>Constructs a {@link StandardTokenizer} filtered by a {@link
/// StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}.
/// </summary>
public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
TokenStream result = new ChineseTokenizer(reader);
result = new StandardFilter(result);
result = new LowerCaseFilter(result);
result = new StopFilter(result, CHINESE_ENGLISH_STOP_WORDS);
return result;
}
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
}
}
ChineseTokenizer类的实现:
这里通过词典来正向匹配字符,返回lucene下定义的token流
using
System;
using
System.Collections.Generic;
using
System.Text;
using
Lucene.Net.Analysis;
using
System.Collections;
using
System.Text.RegularExpressions;
using
System.IO;
![](https://i-blog.csdnimg.cn/blog_migrate/f0cd6c7f9e7ae96feae062cb48f670f0.gif)
namespace
Lucene.Fanswo
![](https://i-blog.csdnimg.cn/blog_migrate/34031c708bfe702fe82d01ff5c6593aa.gif)
{
class ChineseTokenizer : Tokenizer
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
private int offset = 0, bufferIndex = 0, dataLen = 0;//偏移量,当前字符的位置,字符长度
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
private int start;//开始位置
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
/**//// <summary>
/// 存在字符内容
/// </summary>
private string text;
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
/**//// <summary>
/// 切词所花费的时间
/// </summary>
public double TextSeg_Span = 0;
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
/**//// <summary>Constructs a tokenizer for this Reader. </summary>
public ChineseTokenizer(System.IO.TextReader reader)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
this.input = reader;
text = input.ReadToEnd();
dataLen = text.Length;
}
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
/**//// <summary>进行切词,返回数据流中下一个token或者数据流为空时返回null
/// </summary>
///
public override Token Next()
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
Token token = null;
WordTree tree = new WordTree();
//读取词库
tree.LoadDict();
//初始化词库,为树形
Hashtable t_chartable = WordTree.chartable;
string ReWord = "";
string char_s;
start = offset;
bufferIndex = start;
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
while (true)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
//开始位置超过字符长度退出循环
if (start >= dataLen)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
break;
}
//获取一个词
char_s = text.Substring(start, 1);
if (string.IsNullOrEmpty(char_s.Trim()))
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
start++;
continue;
}
//字符不在字典中
if (!t_chartable.Contains(char_s))
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
if (ReWord == "")
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
int j = start + 1;
switch (tree.GetCharType(char_s))
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
case 0://中文单词
ReWord += char_s;
break;
case 1://英文单词
j = start + 1;
while (j < dataLen)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
if (tree.GetCharType(text.Substring(j, 1)) != 1)
break;
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
j++;
}
ReWord += text.Substring(start, j - offset);
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
break;
case 2://数字
j = start + 1;
while (j < dataLen)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
if (tree.GetCharType(text.Substring(j, 1)) != 2)
break;
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
j++;
}
ReWord += text.Substring(start, j - offset);
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
break;
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
default:
ReWord += char_s;//其他字符单词
break;
}
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
offset = j;//设置取下一个词的开始位置
}
else
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
offset = start;//设置取下一个词的开始位置
}
//返回token对象
return new Token(ReWord, bufferIndex, bufferIndex + ReWord.Length - 1);
}
//字符在字典中
ReWord += char_s;
//取得属于当前字符的词典树
t_chartable = (Hashtable)t_chartable[char_s];
//设置下一循环取下一个词的开始位置
start++;
if (start == dataLen)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
offset = dataLen;
return new Token(ReWord, bufferIndex, bufferIndex + ReWord.Length - 1);
}
}
return token;
}
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
}
}
测试的代码:
using
System;
using
System.Collections.Generic;
using
System.Text;
![](https://i-blog.csdnimg.cn/blog_migrate/f0cd6c7f9e7ae96feae062cb48f670f0.gif)
using
Analyzer
=
Lucene.Net.Analysis.Analyzer;
using
SimpleAnalyzer
=
Lucene.Net.Analysis.SimpleAnalyzer;
using
StandardAnalyzer
=
Lucene.Net.Analysis.Standard.StandardAnalyzer;
using
Token
=
Lucene.Net.Analysis.Token;
using
TokenStream
=
Lucene.Net.Analysis.TokenStream;
![](https://i-blog.csdnimg.cn/blog_migrate/f0cd6c7f9e7ae96feae062cb48f670f0.gif)
namespace
MyLuceneTest
![](https://i-blog.csdnimg.cn/blog_migrate/34031c708bfe702fe82d01ff5c6593aa.gif)
{
class Program
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
[STAThread]
public static void Main(System.String[] args)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
try
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
Test("中华人民共和国在1949年建立,从此开始了新中国的伟大篇章。长春市长春节致词", true);
}
catch (System.Exception e)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
System.Console.Out.WriteLine(" caught a " + e.GetType() + " with message: " + e.Message + e.ToString());
}
}
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
internal static void Test(System.String text, bool verbose)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
System.Console.Out.WriteLine(" Tokenizing string: " + text);
Test(new System.IO.StringReader(text), verbose, text.Length);
}
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
internal static void Test(System.IO.TextReader reader, bool verbose, long bytes)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
//Analyzer analyzer = new StandardAnalyzer();
Analyzer analyzer = new Lucene.Fanswo.ChineseAnalyzer();
TokenStream stream = analyzer.TokenStream(null, reader);
System.DateTime start = System.DateTime.Now;
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
int count = 0;
for (Token t = stream.Next(); t != null; t = stream.Next())
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
if (verbose)
![](https://i-blog.csdnimg.cn/blog_migrate/3112b7b6526db5bc83e275260ae60525.gif)
{
System.Console.Out.WriteLine("Token=" + t.ToString());
}
count++;
}
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
System.DateTime end = System.DateTime.Now;
![](https://i-blog.csdnimg.cn/blog_migrate/587e34b10dcf5efbc0859b53470a2db3.gif)
long time = end.Ticks - start.Ticks;
System.Console.Out.WriteLine(time + " milliseconds to extract " + count + " tokens");
System.Console.Out.WriteLine((time * 1000.0) / count + " microseconds/token");
System.Console.Out.WriteLine((bytes * 1000.0 * 60.0 * 60.0) / (time * 1000000.0) + " megabytes/hour");
}
}
}
测试结果:
![](https://i-blog.csdnimg.cn/blog_migrate/31d883556b2f50d38fb1a916473d2ac6.jpeg)
完毕!