在对文本信息进行提取的时候,往往需要对一个数据集的单词进行词频统计,下面代码就实现了这个功能:
package test;
import java.io.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.ansj.domain.Term;
import org.ansj.splitWord.analysis.ToAnalysis;
import org.apache.commons.collections4.map.HashedMap;
import org.apache.tika.Tika;
import org.apache.tika.exception.TikaException;
/**
* 对一个目录下的中文文本进行抽取,并统计词频,按降序对词频进行排序
* @author NS
* @date 2019.4.26
*/
public class ChineseWordFrequence {
public static void main(String[] args) throws Exception {
String filePath = "D:/test/文本分类语料库-2-测试用";//读取的中文文本目录
String resultPath = "C:/Users/user/Desktop/wordFreText.txt";//结果文件的路径
Map<String, Integer> wordFreCount = getWordFrequence(filePath);
saveResult(wordFreCount,resultPath);
System.out.println("文件已读取完毕,词频已保存");
}
//抽取一个目录下的中文文本
private static Map<String, Integer> getWordFrequence(String fileList) throws IOException, TikaException {
System.out.println("开始提取目录下的文本");
File file = new File(fileList);
Map<String, Integer> wordFrequence = new HashedMap<>();
Tika tika = new Tika();
String topic = "";
for(File filelist:file.listFiles()){
if(filelist.isDirectory()){
topic = filelist.getName();
System.out.println(topic);
int textCount = 0;
for(File f:filelist.listFiles()){
String text = tika.parseToString(f);
List<Term> parse = ToAnalysis.parse(text);
for(Term term:parse){
String word = term.getName();
if(!wordFrequence.containsKey(word)){
wordFrequence.put(word, 1);
}else{
wordFrequence.put(word, wordFrequence.get(word) + 1);
}
}
System.out.println(textCount++);
}
}
SortMap(wordFrequence);//按值进行排序
}
return wordFrequence;
}
//按value值的大小进行排序
private static void SortMap(Map<String, Integer> oldmap){
ArrayList<Map.Entry<String, Integer>> list = new ArrayList<Map.Entry<String,Integer>>(oldmap.entrySet());
Collections.sort(list,new Comparator<Map.Entry<String,Integer>>() {
public int compare(Entry<String, Integer> o1,Entry<String, Integer> o2) {
return o2.getValue() - o1.getValue();//降序排列
}
});
for(int i = 0;i < list.size();i++) {
System.out.println(list.get(i).getKey() + ": " +list.get(i).getValue());
}
}
//保存结果
private static void saveResult(Map<String, Integer> wordFreCount,String resultPath) throws FileNotFoundException{
File file = new File(resultPath);
PrintWriter pw = new PrintWriter(resultPath);
for(String str:wordFreCount.keySet()){
pw.write(str + "\t" + wordFreCount.get(str) + "\n");
}
pw.close();
}
}