String hlm = IOUtil.readTxt("红楼梦.txt");
CoreStopWordDictionary.FILTER = new Filter() {
@Override
public boolean shouldInclude(Term term) {
if(term.nature.startsWith("nr")) {//当为词性为nr人名时放行,其他过滤掉
return !CoreStopWordDictionary.contains(term.word);
}else{
return false;
}
}
};
Occurrence occurrence = new Occurrence();
occurrence.addAll(hlm);
occurrence.compute();
Set> uniGram = occurrence.getUniGram();
List> wordList = new ArrayList>(uniGram);
Collections.sort(wordList,new Comparator>() {// 根据value排序
public int compare(Map.Entry o1, Map.Entry o2) {
double result = o2.getValue().getFrequency() - o1.getValue().getFrequency();
if (result > 0)
return 1;
else if (result == 0)
return 0;
else
return -1;
}
});
// wordList = wordList.subList(0, 50);
for (Map.Entry entry : wordList) {
TermFrequency termFrequency = entry.getValue();
System.out.println(termFrequency);
}