Below codes use Object.wait() & Object.notifyAll() to construct the Producer & Consumer model.
There are one producer to read records of a file into a list, then notify threads to write into some seperate files.
Also used bi-buffer to enhance performance.
1. Producer Class : LogReaderDupBuffer
/**
* Copyright : Bian Junjie
*/<pre name="code" class="java">package org.mantis.test;
import java.io.BufferedReader;
//import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
//import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class LogReaderDupBuffer {
private static String FILEPATH = "*/properties/logFilesReceived.txt"; //replace with the right file path
private List<String> buffer1 = new ArrayList<String>();
private List<String> buffer2 = new ArrayList<String>();
private BufferedReader fileInput;
private List<Thread> threads = new ArrayList<Thread>();
private int counter = 0;
private static int MAXLINE = 10000;
public LogReaderDupBuffer() {
try {
fileInput = new BufferedReader(new FileReader(new File(FILEPATH)));
} catch (FileNotFoundException e) {
e.printStackTrace();
}
}
public String putList(int i) throws IOException {
String inputBuffer = null;
while (buffer1.size() <= MAXLINE) {
if ((inputBuffer = fileInput.readLine()) != null) {
buffer1.add(i++, inputBuffer);
} else {
break;
}
}
i = 0;
synchronized (this.buffer2) {
while (buffer1.size() > 0) {
buffer2.add(i, buffer1.remove(0));
this.counter++;
}
buffer2.notifyAll();
}
return inputBuffer;
}
public void manufatoryProduce() throws IOException, InterruptedException {
String inputBuffer = "";
while (inputBuffer != null) {
inputBuffer = putList(0);
// System.out.println(counter);
}
for (Thread t : threads)
t.interrupt();
fileInput.close();
}
public String charToString(char[] buffer) {
if (buffer == null)
return "";
StringBuffer string = new StringBuffer(66);
for (char c : buffer) {
// System.out.print(c);
string.append(c);
}
return string.toString();
}
public void putThread(Thread t) {
this.threads.add(t);
}
public int getCounter() {
return counter;
}
public void setCounter(int counter) {
this.counter = counter;
}
public List<String> getBuffer1() {
return buffer1;
}
public void setBuffer1(List<String> buffer1) {
this.buffer1 = buffer1;
}
public List<String> getBuffer2() {
return buffer2;
}
public void setBuffer2(List<String> buffer2) {
this.buffer2 = buffer2;
}
}
2. Consumer Class : LogConsumerDupBuffer
<pre name="code" class="java"><pre name="code" class="java"><pre name="code" class="java">/**
* Copyright : Bian Junjie
*/
package org.mantis.test;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class LogConsumerDupBuffer implements Runnable {
private List<String> buffer1;
private List<String> buffer2 = new ArrayList<String>();
private BufferedWriter fout;
private int counter;
public LogConsumerDupBuffer() {
}
public LogConsumerDupBuffer(List<String> list, int id) {
this.buffer1 = list;
try {
this.fout = new BufferedWriter(new FileWriter(
"*/properties/logFiles"
+ id + "Received.txt")); // replace with the right file path
} catch (IOException e) {
e.printStackTrace();
}
}
public void processLog(int i) throws InterruptedException, IOException {
synchronized (buffer1) {
while (buffer1.isEmpty()) {
buffer1.wait();
}
while (buffer1.size() > 0) {
buffer2.add(i, buffer1.remove(0));
}
}
while (buffer2.size() > 0) {
fout.write(buffer2.remove(0));
fout.newLine();
counter++;
}
}
@Override
public void run() {
System.out.println("Thread LogConsumer starts...");
while (true) {
try {
processLog(0);
} catch (InterruptedException e) {
try {
fout.close();
} catch (IOException e1) {
e1.printStackTrace();
}
System.out.println("thread" + Thread.currentThread().getName()
+ " stopped, processed " + counter + " lines...");
break;
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
3. Test : UnitTest
<pre name="code" class="java">/**
* Copyright : Bian Junjie
*/
package org.mantis.test;
import java.io.IOException;
public class UnitTest {
public static void producerAndConsumer() {
long startTime = System.currentTimeMillis();
LogReaderDupBuffer lReader = new LogReaderDupBuffer();
Thread t = null;
for (int id = 0; id < 3;) {
// LogConsumer lConsumer = new LogConsumer(lReader.getBuffer2(),
// id);
LogConsumerDupBuffer lConsumer = new LogConsumerDupBuffer(
lReader.getBuffer2(), id);
t = new Thread(lConsumer);
t.setName("thread" + id++);
lReader.putThread(t);
t.start();
}
try {
// Thread.currentThread().sleep(10);
lReader.manufatoryProduce();
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
long endTime = System.currentTimeMillis();
System.out.println("Processed " + lReader.getCounter()
+ " records used time: " + (endTime - startTime));
}
/**
* @param args
*/
public static void main(String[] args) {
UnitTest.producerAndConsumer();
}
}
Test result:
10000
Processed 16268814 records used time: 48125
threadthread0 stopped, processed 4530453 lines...
threadthread1 stopped, processed 7587946 lines...
threadthread2 stopped, processed 4150415 lines...