分词器实现代码:
package com.zd.tokenizer;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.*;
public class Tokenizer {
private Map<Character, Object> dictionary;
public Tokenizer(String dictionaryFilePath) throws IOException {
//红黑树的实现
dictionary = new TreeMap<>();
//从文件加载字典到TreeMap
this.loadDictionary(dictionaryFilePath);
}
private void loadDictionary(String dictionaryFilePath) throws IOException {
BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(dictionaryFilePath)));
String line = null;
while ((line = reader.readLine()) != null) {
line = line.trim();
if (line.length() == 0) {
continue;
}
char c;
Map<Character, Object> child = this.dictionary;
//组成以这个字符开头的词的树
for (int i = 0; i < line.length(); i++) {
c = line.charAt(i);
Map<Character, Object> ccMap = (Map<Character, Object>) child.get(c);
if (ccMap == null) {
ccMap = new HashMap<Character, Object>();
child.put(c, ccMap);
}
child = ccMap;
}
child.put(' ', null);
}
}
public List<String> participie(String text) {
if (text == null) {
return null;
}
text = text.trim();
if (text.length() == 0) {
return null;
}
List<String> tokens = new ArrayList<>();
char c;
for (int i = 0; i < text.length(); ) {
StringBuilder token = new StringBuilder();
Map<Character, Object> child = this.dictionary;
boolean matchToken = false;
for (int j = i; j < text.length(); j++) {
c = text.charAt(j);
Map<Character, Object> ccMap = (Map<Character, Object>) child.get(c);
if (ccMap == null) {
if (child.containsKey(' ')) {
matchToken = true;
i = j;
}
break;
} else {
token.append(c);
child = ccMap;
}
}
//匹配到词
if (matchToken) {
tokens.add(token.toString());
} else {
if (child.containsKey(' ')) {//短的也是词,如张三丰,张三
tokens.add(token.toString());
break;
} else {//没有匹配到词,则该字符作为一个词
tokens.add("" + text.charAt(i));
i++;
}
}
}
return tokens;
}
public static void main(String[] args) throws IOException {
Tokenizer tk = new Tokenizer(Tokenizer.class.getResource("/dictionary.txt").getPath());
List<String> tokens = tk.participie("想过离开,以这种方式存在,是因为那些旁白那些姿态那些伤害");
for (String s : tokens){
System.out.println(s);
}
}
}