利用java实现对一个目录下中文文本的抽取，并统计词频，降序排序。

在对文本信息进行提取的时候，往往需要对一个数据集的单词进行词频统计，下面代码就实现了这个功能：

package test;
import java.io.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.ansj.domain.Term;
import org.ansj.splitWord.analysis.ToAnalysis;
import org.apache.commons.collections4.map.HashedMap;
import org.apache.tika.Tika;
import org.apache.tika.exception.TikaException;
/**
 * 对一个目录下的中文文本进行抽取，并统计词频,按降序对词频进行排序
 * @author NS
 * @date 2019.4.26
 */
public class ChineseWordFrequence {
	public static void main(String[] args) throws Exception {
		String filePath = "D:/test/文本分类语料库-2-测试用";//读取的中文文本目录
		String resultPath = "C:/Users/user/Desktop/wordFreText.txt";//结果文件的路径
		Map<String, Integer> wordFreCount = getWordFrequence(filePath);
		saveResult(wordFreCount,resultPath);
		System.out.println("文件已读取完毕，词频已保存");
	}
	
	//抽取一个目录下的中文文本
	private static Map<String, Integer> getWordFrequence(String fileList) throws IOException, TikaException {
		System.out.println("开始提取目录下的文本");
		File file = new File(fileList);
		Map<String, Integer> wordFrequence = new HashedMap<>();

		Tika tika = new Tika();
		String topic = "";
		for(File filelist:file.listFiles()){
			if(filelist.isDirectory()){
				topic = filelist.getName();
				System.out.println(topic);
				
				int textCount = 0;
				for(File f:filelist.listFiles()){
					String text = tika.parseToString(f);
					List<Term> parse = ToAnalysis.parse(text);

					for(Term term:parse){
						String word = term.getName();
						if(!wordFrequence.containsKey(word)){
							wordFrequence.put(word, 1);
						}else{
							wordFrequence.put(word, wordFrequence.get(word) + 1);
						}
					}
					System.out.println(textCount++);
				}
				
			}
			SortMap(wordFrequence);//按值进行排序
		}
		return wordFrequence;
	}

	//按value值的大小进行排序
	private static void SortMap(Map<String, Integer> oldmap){
		ArrayList<Map.Entry<String, Integer>> list = new ArrayList<Map.Entry<String,Integer>>(oldmap.entrySet());
		Collections.sort(list,new Comparator<Map.Entry<String,Integer>>() {
			public int compare(Entry<String, Integer> o1,Entry<String, Integer> o2) {
				return o2.getValue() - o1.getValue();//降序排列
			}
		});

		for(int i = 0;i < list.size();i++) {
			System.out.println(list.get(i).getKey() + ": " +list.get(i).getValue());
		}
	}

	//保存结果
	private static void saveResult(Map<String, Integer> wordFreCount,String resultPath) throws FileNotFoundException{
		File file = new File(resultPath);
		PrintWriter pw = new PrintWriter(resultPath);
		for(String str:wordFreCount.keySet()){
			pw.write(str + "\t" + wordFreCount.get(str) + "\n");
		}
		pw.close();
	}
}