为了比较beansdb,写了一个基于socket的java文件读取测试。
server:
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PushbackInputStream;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import org.apache.commons.io.FileUtils;
import com.taobao.tlog.util.NamedThreadFactory;
public class FileServer {
public void run() throws IOException {
bosses.submit(new Boss());
}
static ExecutorService bosses = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("boss", false));
static ExecutorService workers = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("worker", false));
private static class Boss implements Callable {
@Override
public Object call() throws Exception {
ServerSocket server = new ServerSocket(8080);
while (true) {
Socket socket = server.accept();
workers.submit(new Wroker(socket));
}
}
}
private static class Wroker implements Callable {
Socket socket;
PushbackInputStream input;
public Wroker(Socket socket) throws IOException {
this.socket = socket;
input = new PushbackInputStream(socket.getInputStream(), 1);
}
protected void receive(String path) throws IOException {
File file = new File(path);
byte[] data = FileUtils.readFileToByteArray(file);
OutputStream output = socket.getOutputStream();
output.write(data);
output.flush();
System.out.println(path+" flush");
}
public void accept() throws Exception {
int bytesToRead = 0;
while (true) {
bytesToRead = input.available();
if (bytesToRead > 0) {
byte[] t = new byte[45];
input.read(t);
String path = new String(t);
// System.out.println("path=" + path);
receive(path);
} else {
int b = input.read(); // 此操作会阻塞,直到有数据被读到
if (b < 0) {
throw new IOException(
" end of the socket input stream has been reached,may be server socket is closed!");
} else {
input.unread(b);
continue;
}
}
}
}
@Override
public Object call() {
while (true) {
try {
accept();
} catch (Exception e) {
e.printStackTrace();
System.exit(-1);
}
}
}
}
public static void main(String[] args) throws IOException {
new FileServer().run();
}
}
client
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PushbackInputStream;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.text.DecimalFormat;
import java.util.Random;
public class FileClient {
static Random random = new Random();
Socket socket;
PushbackInputStream input;
public FileClient() throws IOException {
init();
}
public void init() throws IOException {
socket = new Socket();
socket.setReuseAddress(true);
socket.setKeepAlive(true);
socket.setTcpNoDelay(true);
socket.setSoLinger(true, 0);
socket.setSoTimeout(60000);
socket.connect(new InetSocketAddress("localhost", 8080), 60000);
input = new PushbackInputStream(socket.getInputStream(), 1);
}
public void send(String path) throws IOException {
// System.out.println(path);
OutputStream out = socket.getOutputStream();
OutputStreamWriter writer = new OutputStreamWriter(out);
writer.write(path);
writer.flush();
}
/**
* System.out.println("readed:" + readBytes + ",bytesToRead:" + bytesToRead);
* <pre>
* readed:-1,bytesToRead:0
* readed:-1,bytesToRead:180224
* readed:131072,bytesToRead:376831
* readed:262144,bytesToRead:344063
* readed:393216,bytesToRead:324300
* readed:524288,bytesToRead:193228
* readed:655360,bytesToRead:62156
* readed:717516,bytesToRead:0
* </pre>
* @throws IOException
*/
public void readLine() throws IOException {
int bytesToRead = 0;
int readBytes = -1;
while (true) {
bytesToRead = input.available();
if (readBytes >= 717516) {
break;
}
if (bytesToRead > 0) {
byte[] data = new byte[bytesToRead];
readBytes += input.read(data);
} else {
int b = input.read(); // 此操作会阻塞,直到有数据被读到
if (b < 0) {
throw new IOException(
" end of the socket input stream has been reached,may be server socket is closed!");
} else {
input.unread(b);
continue;
}
}
}
}
public void test() throws IOException {
DecimalFormat df = new DecimalFormat("#00");
long begin = System.currentTimeMillis();
for (int i = 0; i < 100; i++) {
int id = random.nextInt(10000);
int month = id % 12 + 1;
int day = id % 30 + 1;
int ix = id % 99 + 1;
String file = "/duitang/data/pic_test/2013/" + df.format(month) + "/" + df.format(day) + "/HDFS"
+ df.format(ix) + ".jpeg";
send(file);
readLine();
}
long end = System.currentTimeMillis();
System.out.println((end - begin) + "ms ");
}
public static void main(String[] args) throws IOException {
FileClient client = new FileClient();
for (int i = 0; i < 10; ++i) {
client.test();
}
}
}
测试数据准备,测试数据按照/year/month/day/data 存放,一共36000张图片:
import java.io.File;
import java.io.IOException;
import java.text.DecimalFormat;
import org.apache.commons.io.FileUtils;
public class FileUtil {
public static void buildData() throws IOException {
DecimalFormat format = new DecimalFormat("#00");
String path = "/duitang/data/pic_test/2013";
for (int month = 1; month < 13; ++month) {
File mongthPath = new File(path, format.format(month));
mongthPath.mkdirs();
for (int day = 1; day < 31; ++day) {
File dayPath = new File(mongthPath, format.format(day));
dayPath.mkdirs();
for (int i = 1; i < 100; ++i) {
FileUtils.copyFile(new File("/tmp/HDFS.jpeg"), new File(dayPath, "HDFS" + format.format(i)
+ ".jpeg"));
}
}
}
}
}
在我笔记本上的性能:
187ms
174ms
157ms
169ms
171ms
154ms
168ms
155ms
163ms
164ms
产生的图片请求,确保是随机的:
/duitang/data/pic_test/2013/03/27/HDFS48.jpeg flush
/duitang/data/pic_test/2013/09/09/HDFS39.jpeg flush
/duitang/data/pic_test/2013/02/08/HDFS74.jpeg flush
/duitang/data/pic_test/2013/06/12/HDFS87.jpeg flush
/duitang/data/pic_test/2013/08/08/HDFS50.jpeg flush
/duitang/data/pic_test/2013/01/07/HDFS79.jpeg flush
/duitang/data/pic_test/2013/10/16/HDFS52.jpeg flush
/duitang/data/pic_test/2013/02/08/HDFS47.jpeg flush
/duitang/data/pic_test/2013/07/19/HDFS88.jpeg flush
/duitang/data/pic_test/2013/05/29/HDFS14.jpeg flush
/duitang/data/pic_test/2013/08/20/HDFS71.jpeg flush
/duitang/data/pic_test/2013/08/20/HDFS26.jpeg flush
/duitang/data/pic_test/2013/03/09/HDFS45.jpeg flush
/duitang/data/pic_test/2013/03/15/HDFS36.jpeg flush
/duitang/data/pic_test/2013/09/21/HDFS39.jpeg flush
/duitang/data/pic_test/2013/11/23/HDFS53.jpeg flush
/duitang/data/pic_test/2013/04/10/HDFS64.jpeg flush
/duitang/data/pic_test/2013/02/26/HDFS80.jpeg flush
/duitang/data/pic_test/2013/09/27/HDFS06.jpeg flush
/duitang/data/pic_test/2013/01/07/HDFS46.jpeg flush
/duitang/data/pic_test/2013/04/16/HDFS19.jpeg flush
/duitang/data/pic_test/2013/10/10/HDFS88.jpeg flush
/duitang/data/pic_test/2013/05/11/HDFS29.jpeg flush
/duitang/data/pic_test/2013/08/14/HDFS86.jpeg flush
/duitang/data/pic_test/2013/07/07/HDFS85.jpeg flush
/duitang/data/pic_test/2013/09/21/HDFS51.jpeg flush
disable linux cache后:
普通FS
326 ms
286 ms
303 ms
292 ms
296 ms
283 ms
292 ms
290 ms
273 ms
289 ms
beansdb
368.748903275 ms
326.793909073 ms
337.444067001 ms
322.673082352 ms
336.313009262 ms
331.909179688 ms
329.982995987 ms
275.442838669 ms
271.515130997 ms
333.788871765 ms
基于BufferedRandomAccess
127ms
125ms
125ms
123ms
127ms
126ms
125ms
128ms
127ms
126ms