java中一个队列多个线程存取,Java 多线程写同一个文件实现 | 学步园

最近项目中需要从网站上抓取大量的数据，采用了多线程技术，每个线程抓取的数据都需要保存到一个文件中，避免消耗大量的内存。

思路：多个访问线程将需要写入到文件中的数据先保存到一个队列里面，然后由专门的 写出线程负责从队列中取出数据并写入到文件中。

WriterQueue.java 存放要输出的数据队列

package com.yulore.write;

import java.util.LinkedList;

import java.util.concurrent.locks.Condition;

import java.util.concurrent.locks.Lock;

import java.util.concurrent.locks.ReentrantLock;

public class WriterQueue {

private static final int MAX_QUEUE_SIZE = 5000;

private LinkedList queue = new LinkedList();

private Lock lock = new ReentrantLock();

private Condition notFull = lock.newCondition();

private Condition notEmpty = lock.newCondition();

private static WriterQueue manager = new WriterQueue();

private WriterQueue(){

}

public static WriterQueue getQueue(){

return manager;

}

public void put(String phone){

lock.lock();

try {

while (queue.size() == MAX_QUEUE_SIZE) {

System.out.println("warning: data queue is full!");

notFull.await();

}

queue.addFirst(phone);

notEmpty.signal();

} catch (InterruptedException e) {

e.printStackTrace();

} finally{

lock.unlock();

}

}

public LinkedList takeAll(){

LinkedList retVal = new LinkedList();

lock.lock();

try {

while (queue.size() == 0) {

System.out.println("warning: data queue is empty!");

notEmpty.await();

}

retVal.addAll(queue);

//for(String str : queue){

//retVal.add(str);

//}

//清空队列

queue.clear();

notFull.signal();

} catch (InterruptedException e) {

e.printStackTrace();

} finally{

lock.unlock();

}

return retVal;

}

}

WriteTask_New.java 模拟产生数据的线程类

package com.yulore.write;

public class WriteTask_New implements Runnable {

@Override

public void run() {

for(int i=0;i<20;i++){

//try {

//sleep(100);

//} catch (InterruptedException e) {

//e.printStackTrace();

//}

WriterQueue.getQueue().put("for:"+i+" thread:"+Thread.currentThread().getName());

}

}

private void sleep(int millis) throws InterruptedException {

Thread.sleep(millis);

}

}

OutputTask.java 负责将数据写入到文件中

package com.yulore.write;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStreamWriter;

import java.util.LinkedList;

public class OutputTask implements Runnable {

private String fileName;

public OutputTask(String fileName) {

this.fileName = fileName;

}

@Override

public void run() {

while(true){

try {

sleep(5000);

} catch (InterruptedException e) {

e.printStackTrace();

}

LinkedList list = WriterQueue.getQueue().takeAll();

write2Disk(list);

list = null;

}

}

private void write2Disk(LinkedList list) {

if(list==null ||list.size()==0){

System.out.println("no data...");

return;

}

System.out.println("开始序列化数据 "+fileName);

String path = "D:/fbb/myWorkSpace_DW07/";

File outputFile = new File(path+fileName);

if(outputFile==null ||!outputFile.exists()){

try {

outputFile.createNewFile();

} catch (IOException e) {

e.printStackTrace();

}

}

FileOutputStream out = null;

OutputStreamWriter writer = null;

BufferedWriter bw = null;

try {

out = new FileOutputStream(outputFile, true);

writer = new OutputStreamWriter(out);

bw = new BufferedWriter(writer);

for(String content : list){

bw.write(content);

bw.newLine();

bw.flush();

}

} catch (IOException e) {

e.printStackTrace();

}finally{

try {

if(bw!=null)

bw.close();

} catch (IOException e) {

e.printStackTrace();

}

}

}

private void sleep(int millis) throws InterruptedException {

Thread.sleep(millis);

}

}

测试类

package com.yulore.write;

public class TestWrite {

/**

* @param args

*/

public static void main(String[] args) {

//test();

test02();

}

private static void test02() {

WriteTask_New write = new WriteTask_New();

for(int i=0;i<4;i++){

new Thread(write).start();

}

OutputTask output = new OutputTask("abc.txt");

new Thread(output).start();

}

private static void test() {

WriteTask write = new WriteTask("abc.txt");

for(int i=0;i<5;i++){

new Thread(write).start();

}

}

}