用ForkJoin框架实现的统计词频和一般单线程统计的性能比较
源码如下:
- Folder.java
package ForkJoin;
import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
/**
* Folder 是磁盘文件夹模型
* @author TreeNode
*
*/
public class Folder {
// 子文件夹
private final List<Folder> subFolders;
private final List<Document> documents;
Folder(List<Folder> subFolders, List<Document> documents) {
this.subFolders = subFolders;
this.documents = documents;
}
public List<Folder> getSubFolders() {
return subFolders;
}
public List<Document> getDocuments() {
return documents;
}
static Folder fromDirectory(File dir) throws IOException {
List<Document> documents = new LinkedList<>();
List<Folder> subFolders = new LinkedList<>();
for (File entry : dir.listFiles()) {
if (entry.isDirectory()) {
subFolders.add(Folder.fromDirectory(entry));
} else {
documents.add(Document.fromFile(entry));
}
}
return new Folder(subFolders, documents);
}
}
- Document.java
package ForkJoin;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
public class Document {
private final List<String> lines;
public Document(List<String> lines) {
this.lines = lines;
}
public List<String> getLines() {
return lines;
}
static Document fromFile(File file) throws FileNotFoundException, IOException{
// 是有序的
List<String> lines = new LinkedList<>();
try(BufferedReader reader = new BufferedReader(new FileReader(file))){
String line = reader.readLine();
while(line != null){
lines.add(line);
line = reader.readLine();
}
}
return new Document(lines);
}
}
- WordCounter.java
package ForkJoin;
import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.RecursiveTask;
public class WordCounter {
// 分词
String[] wordsIn(String line) {
return line.trim().split("(\\s|\\p{Punct})+");
}
// 统计
Long occurrencesCount(Document document, String searchedWord) {
long count = 0;
for (String line : document.getLines()) {
for (String word : wordsIn(line)) {
if (searchedWord.equals(word)) {
count = count + 1;
}
}
}
return count;
}
/**
* 单线程统计,递归遍历文件夹,分别统计每个文件夹下文件中词频
* @param folder 搜索的文件夹
* @param searchedWord 要搜索的单词
* @return
*/
Long countOccurrencesOnSingleThread(Folder folder, String searchedWord) {
long count = 0;
for (Folder subFolder : folder.getSubFolders()) {
count = count + countOccurrencesOnSingleThread(subFolder, searchedWord);
}
for (Document document : folder.getDocuments()) {
count = count + occurrencesCount(document, searchedWord);
}
return count;
}
/**
* 在文件中搜索词频
* @author TreeNode
*
*/
class DocumentSearchTask extends RecursiveTask<Long> {
private final Document document;
private final String searchedWord;
DocumentSearchTask(Document document, String searchedWord) {
super();
this.document = document;
this.searchedWord = searchedWord;
}
@Override
protected Long compute() {
return occurrencesCount(document, searchedWord);
}
}
/**
* 文件夹搜索任务
* @author TreeNode
*
*/
class FolderSearchTask extends RecursiveTask<Long> {
private final Folder folder;
private final String searchedWord;
FolderSearchTask(Folder folder, String searchedWord) {
super();
this.folder = folder;
this.searchedWord = searchedWord;
}
@Override
protected Long compute() {
long count = 0L;
List<RecursiveTask<Long>> forks = new LinkedList<>();
// 拆分任务
for (Folder subFolder : folder.getSubFolders()) {
FolderSearchTask task = new FolderSearchTask(subFolder, searchedWord);
forks.add(task);
task.fork();
}
// 拆分任务
for (Document document : folder.getDocuments()) {
DocumentSearchTask task = new DocumentSearchTask(document, searchedWord);
forks.add(task);
task.fork();
}
// 合并结果
for (RecursiveTask<Long> task : forks) {
count = count + task.join();
}
// 返回词数目
return count;
}
}
/**
* 多任务统计
*/
private final ForkJoinPool forkJoinPool = new ForkJoinPool();
Long countOccurrencesInParallel(Folder folder, String searchedWord) {
return forkJoinPool.invoke(new FolderSearchTask(folder, searchedWord));
}
public static void main(String[] args) throws IOException {
// 单词统计对象
WordCounter wordCounter = new WordCounter();
// 生成Folder对象
Folder folder = Folder.fromDirectory(new File("E:\\eclipse38_wp\\c6713_gpio_test"));
// 重复统计次数
final int repeatCount = Integer.decode("8");
long counts;
long startTime;
long stopTime;
long[] singleThreadTimes = new long[repeatCount];
long[] forkedThreadTimes = new long[repeatCount];
for (int i = 0; i < repeatCount; i++) {
startTime = System.currentTimeMillis();
// 单线程统计 device 出现的次数
counts = wordCounter.countOccurrencesOnSingleThread(folder,"device");
stopTime = System.currentTimeMillis();
singleThreadTimes[i] = (stopTime - startTime);
System.out.println(counts + " , single thread search took " + singleThreadTimes[i]
+ "ms");
}
for (int i = 0; i < repeatCount; i++) {
startTime = System.currentTimeMillis();
counts = wordCounter.countOccurrencesInParallel(folder, "device");
stopTime = System.currentTimeMillis();
forkedThreadTimes[i] = (stopTime - startTime);
System.out
.println(counts + " , fork / join search took " + forkedThreadTimes[i] + "ms");
}
System.out.println("\nCSV Output:\n");
System.out.println("Single thread,Fork/Join");
for (int i = 0; i < repeatCount; i++) {
System.out.println(singleThreadTimes[i] + "," + forkedThreadTimes[i]);
}
System.out.println();
}
}
- 输出结果
10 , single thread search took 147ms
10 , single thread search took 35ms
10 , single thread search took 12ms
10 , single thread search took 10ms
10 , single thread search took 12ms
10 , single thread search took 10ms
10 , single thread search took 11ms
10 , single thread search took 11ms
10 , fork / join search took 8ms
10 , fork / join search took 12ms
10 , fork / join search took 9ms
10 , fork / join search took 6ms
10 , fork / join search took 8ms
10 , fork / join search took 6ms
10 , fork / join search took 7ms
10 , fork / join search took 8ms
CSV Output:
Single thread,Fork/Join
147,8
35,12
12,9
10,6
12,8
10,6
11,7
11,8