Analyzer analyzer = new StandardAnalyzer();
TokenStream tokenStream = analyzer.TokenStream("", new StringReader("北京,Hi欢迎你们大家"));
Lucene.Net.Analysis.Token token = null;
while ((token = tokenStream.Next()) != null)
{
Console.WriteLine(token.TermText());
}