java 统计10000篇文章中不同单词出现的次数并以次序排序

统计10000篇文章中不同单词出现的次数并以次序排序

此次统计从两方面入手：一是单线程读取10000个文件；二是打开10000个线程，每个线程读取一个文件

单线程程序

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

public class CountLetters {
//	public static String FILE_NAME = "/home/taohao/temp/english.txt";
	public static String FILE_PATH = "/home/taohao/temp/test";
	private String[] readFromeFile(String[] fileNames){
		String lineString = new String();
		StringBuilder line = new StringBuilder();
		BufferedReader bufferedReader = null;
		for(String name : fileNames){
			try {
				name = FILE_PATH + "/" + name;
				File file = new File(name);
				bufferedReader = new BufferedReader(new FileReader(file));
				while((lineString=bufferedReader.readLine())!=null){
					line.append(lineString);
				}
				bufferedReader.close();
				
			} catch (Exception e) {
				// TODO: handle exception
				System.out.println(e);
			}
		}
//		System.out.println(line.toString());
		String[] letters = line.toString().split("\\W+");
		return letters;
	}
	
	private HashMap<String, Integer> countMap(String[] string){
		HashMap<String, Integer> hashMap = new HashMap<String, Integer>();
		Integer integer = new Integer(1);
		int newIntValue;
		Integer newInteger = null;
		Integer value = null;
		for(String item : string){
			if((value=hashMap.get(item))==null){
				hashMap.put(item, integer);
			}
			else {
				newIntValue = value.intValue() + 1;
				newInteger = new Integer(newIntValue);
				hashMap.put(item, newInteger);
			}
		}
		return hashMap;
	}
	
	// treemap 的key 不能重复，否则会被覆盖，因此此处的转换有问题
//	private TreeMap<Integer, String>hashMapToTreeMap(HashMap<String, Integer> hashMap){
//		TreeMap<Integer, String> treeMap = new TreeMap<Integer, String>(new Comparator<Integer>() {
//
//			@Override
//			public int compare(Integer o1, Integer o2) {
//				// TODO Auto-generated method stub
//				return ~o1.compareTo(o2)+1;
//			}
//		});
//		for(String letter : hashMap.keySet()){
//			treeMap.put(hashMap.get(letter), letter);
//		}
//		return treeMap;
//	}
	
	public static void main(String[] args){
		/*
		 * 单线程
		 */
		long start = System.currentTimeMillis();  //ms
		CountLetters countLetters = new CountLetters();
		File file = new File(CountLetters.FILE_PATH);
		String[] fileNames = file.list();
		String[] letters = countLetters.readFromeFile(fileNames);
		System.out.println(letters.length);
		HashMap<String, Integer> hashMap = countLetters.countMap(letters);
		System.out.println(hashMap.size());
		SortByValues sortByValues = new SortByValues();
		List<Map.Entry<String, Integer>> mapItemList = new ArrayList<>(hashMap.entrySet());
		Collections.sort(mapItemList, sortByValues);
		System.out.println(mapItemList.size());
		System.out.println(mapItemList.toString());
		long end = System.currentTimeMillis();
		System.out.println("start-end=" + (end-start));
	}
}

多线程程序

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

public class CountLettersThreads {
	public static String FILE_PATH = "/home/taohao/temp/test";
//	private String[] fileLiStrings = null;
	public HashMap<String, Integer> hashMap = new HashMap<String, Integer>();
	
	class countThread implements Runnable{
		private String fileName = null;
		private String[] letterStrings = null;
		public countThread(String fileName) {
			// TODO Auto-generated constructor stub
			this.fileName = FILE_PATH + "/" + fileName;
		}
		@Override
		public void run() {
			// TODO Auto-generated method stub
			letterStrings = readFromeFile(fileName);
			synchronized (hashMap) {
				Integer integer = new Integer(1);
				int newIntValue;
				Integer newInteger = null;
				Integer value = null;
				for(String item : letterStrings){
					if((value=hashMap.get(item))==null){
						hashMap.put(item, integer);
					}
					else {
						newIntValue = value.intValue() + 1;
						newInteger = new Integer(newIntValue);
						hashMap.put(item, newInteger);
					}
				}
			}
		}
		
		public String[] readFromeFile(String fileName){
			String lineString = new String();
			StringBuilder line = new StringBuilder();
			File file = new File(fileName);
			BufferedReader bufferedReader = null;
			try {
				bufferedReader = new BufferedReader(new FileReader(file));
				while((lineString=bufferedReader.readLine())!=null){
					line.append(lineString);
				}
				bufferedReader.close();
				
			} catch (Exception e) {
				// TODO: handle exception
				System.out.println("readfile exception " + e);
			}
			System.out.println(line.toString());
			String[] letters = line.toString().split("\\W+");
			return letters;
		}
	}
	
	public static void main(String[] args) throws Exception{
		long start = System.currentTimeMillis();  // ms
		File file = new File(CountLettersThreads.FILE_PATH);
		CountLettersThreads countLettersThreads = new CountLettersThreads();
		String[] fileList = file.list();
		int fileCount = fileList.length;
		for(int i=0; i<fileCount; i++){
			CountLettersThreads.countThread runnaThread = countLettersThreads.new countThread(fileList[i]);
			Thread thread = new Thread(runnaThread);
			thread.start();
			thread.join();
		}
		HashMap<String, Integer> hashMap = countLettersThreads.hashMap;
		System.out.println(hashMap.size());
		SortByValues sortByValues = new SortByValues();
		List<Map.Entry<String, Integer>> mapItemList = new ArrayList<>(hashMap.entrySet());
		Collections.sort(mapItemList, sortByValues);
		System.out.println(mapItemList.toString());
		long end = System.currentTimeMillis();
		System.out.println("start-end=" + (end-start));
 	}
}

另外用到的一个比较类

import java.awt.List;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;

public class SortByValues implements Comparator<Map.Entry<String, Integer>>{
	@Override
	public int compare(Entry<String, Integer> o1, Entry<String, Integer> o2) {
		// TODO Auto-generated method stub
		return ~o1.getValue().compareTo(o2.getValue()) + 1;
	}
}

得到的统计数据为：

单线程用时84537ms， 差不多是多线程的三倍多。

多线程用时31516ms

由上可知两者的差别只有三倍左右的差别，我觉得可能的原因是在最后排序都是统一排序的，

多线程只负责了读文件和区分不同的单词及单词出现的数量，并没有进行局部排序。

如果在每个线程里面进行了局部排序应该两者所用的时间还会继续拉大。

另外本程序中对hashmap的value进行排序，此时是将hashmap的entry作为list的元素，通过collection的sort()函数进行排序的。

collection的sort()函数怎样排序的，我还要进一步查阅资料学习。？？？？