Java multi-threads to read DB then writting to local disk with Producer & Consumer model

原创 2015年07月10日 10:05:49

There will be several Producers to access DB and put results into list; only one Consumer accesses that list to write into local disk.

JDK - 1.5

  1. Producer
package com.mantis.signature.util;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.MissingResourceException;
import java.util.Properties;

import com.mantis.signature.model.table.SignatureImage;

public class ImageDumper implements Runnable{

    private String loginId = "";
    private String loginPwd = "";
    private String signatureDBURL = "";
    private String OSkeyPath = "";
    private String DBSCHEMA = "";
    private int accIndex = 0;
    private String tableName = "";
    private WFEncrypt encryption = null;
    public static byte endOfProcess = 0x00000001;

    public ImageDumper() {}
    public ImageDumper(String userid, String pwd, String tableName) {
        super();
        this.loginId = userid;
        this.loginPwd = pwd;
        this.tableName = tableName;

        String MAPPINGS = "Miscellaneous.properties";
        try {
            Properties props = new Properties();
            props.load(ImageDumper.class.getResourceAsStream(MAPPINGS));
            signatureDBURL = props.getProperty("signaturedburl");
            OSkeyPath = props.getProperty("OSSecretKeyPath");
            DBSCHEMA = props.getProperty("DBSCHEMA");
            System.out.println("DBSCHEMA = " + DBSCHEMA);
            encryption = new WFEncrypt(this.OSkeyPath);
        } catch (IOException ioe) {
            throw new MissingResourceException("Could not load file: " + MAPPINGS, "Oscar", MAPPINGS);
        }
    }

//    @Override
    public void run() {
        Connection conn = null;
        PreparedStatement pstmt = null;
        try {
            Class.forName("com.ibm.db2.jcc.DB2Driver");
            conn = DriverManager.getConnection(signatureDBURL, loginId, loginPwd);
            String selectSQL = "select * from " + this.DBSCHEMA + "." + tableName + " where customer_id in (?,?,?,?,?)";
            pstmt = conn.prepareStatement(selectSQL);
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (SQLException e) {
            e.printStackTrace();
        }
        List custIds = new ArrayList();
        List signatureImgList = null;
        OscarLogger.logMessage(1,"",Thread.currentThread().getName() + "started...");
        System.out.println(Thread.currentThread().getName() + "started...");

        int j;
        while (true) {
            synchronized (ImageCompressUtil.ACCOUNTMAP) {
                for (j = 0; j < 5 && accIndex < ImageCompressUtil.ACCOUNTMAP.size(); j++) {
                    String acct = ((String) ImageCompressUtil.ACCOUNTMAP.get(accIndex++)).trim();
                    custIds.add(acct);
                }
                if(j == 0 && accIndex >= ImageCompressUtil.ACCOUNTMAP.size()){
                    endOfProcess = (byte) (endOfProcess << 1);
                    break;
                }
            }
            signatureImgList = LoadImage(custIds, pstmt);
            synchronized(ImageWriter.imageLst){
                while (!signatureImgList.isEmpty()){
                    ImageWriter.imageLst.add(signatureImgList.remove(0));
                }
            }
        }
        OscarLogger.logMessage(1,"",Thread.currentThread().getName() + "stopped... " + new Date(System.currentTimeMillis()).toString() + " accIndex = " + accIndex);
        System.out.println(Thread.currentThread().getName() + "stopped... " + new Date(System.currentTimeMillis()).toString() + " accIndex = " + accIndex);
    }

    public List LoadImage(List acct, PreparedStatement pstmt) {
        ResultSet rs = null;
        SignatureImage sigImage = null;
        List signatureImgList = new ArrayList();
        try {
            int i = 0;
            while(!acct.isEmpty()){
                pstmt.setString(i+1, (String)acct.remove(0));
                i++;
            }
//            System.out.println("i = " +i);
            if(i < 5){
                for(int j = i ; j < 5; j++){
                    pstmt.setString(j+1, "");
                }
            }
            rs = pstmt.executeQuery();

            while (rs.next()) {
                sigImage = new SignatureImage();
                ...doing object population
                signatureImgList.add(sigImage);
            }

        } catch (SQLException e1) {
            System.out.println(Thread.currentThread().getName());
            e1.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                if (rs != null)
                    rs.close();
            } catch (Exception e) {
            }
        }
        return signatureImgList;
    }
    /**
     * @return the loginId
     */
    public String getLoginId() {
        return loginId;
    }

    /**
     * @param loginId the loginId to set
     */
    public void setLoginId(String loginId) {
        this.loginId = loginId;
    }

    /**
     * @return the loginPwd
     */
    public String getLoginPwd() {
        return loginPwd;
    }

    /**
     * @param loginPwd the loginPwd to set
     */
    public void setLoginPwd(String loginPwd) {
        this.loginPwd = loginPwd;
    }

    /**
     * @return the tableName
     */
    public String getTableName() {
        return tableName;
    }

    /**
     * @param tableName the tableName to set
     */
    public void setTableName(String tableName) {
        this.tableName = tableName;
    }
}
  1. Consumer
package com.mantis.signature.util;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import com.mantis.signature.model.table.SignatureImage;

public class ImageWriter implements Runnable{
    public static List imageLst = new ArrayList();
    private long uncompressedList = 0;
    private long misUncompressedList = 0;
    private String dirName = "";
    private String tableName = "";
    private long size = 0;

    public ImageWriter(String dirname, String tablename){
        this.dirName = dirname;
        this.tableName = tablename;
    }

//    @Override
    public void run() {
        OscarLogger.logMessage(1,"",Thread.currentThread().getName() + "started...");
        System.out.println(Thread.currentThread().getName() + "started...");
        while (true) {
            synchronized (imageLst) {
                if (!imageLst.isEmpty()) {
                    size += imageLst.size();
//                    System.out.println(Thread.currentThread().getName() + " imageLst size = " + size);
                    outPutImageFile(this.dirName,imageLst);
                }
            }
            if(ImageDumper.endOfProcess == ImageCompressUtil.THREADCOUNT && imageLst.isEmpty())
                break;
        }
        System.out.println(Thread.currentThread().getName() + "stopped... " + new Date(System.currentTimeMillis()).toString());
        OscarLogger.logMessage(1, "", Thread.currentThread().getName() + " finished - " + new Date(System.currentTimeMillis()).toString() + " - SIGNATURE DATA DOWNLOAND FROM " + this.tableName + " FINISHED: [[" + uncompressedList + "]]   OK.   [[" + misUncompressedList + "]]   FAILURE.   TOTAL DATA [[" + size + "]]");
    }

    private void outPutImageFile(String dirName, List outImageList) {
        OutputStream out = null;
        try {
            SignatureImage image = null;
            while(!outImageList.isEmpty()){
                image = (SignatureImage) outImageList.remove(0);
                String imageFile = dirName + image.getCustomerId().trim() + "_" + image.getImageId() +".bmp";
                out = new FileOutputStream(imageFile);
                try {
                    out.write(...);
                    OscarLogger.logMessage(1, "Customer Id: " + image.getCustomerId(), "   [OK]");
                    uncompressedList++;
                } catch (Exception e) {
                    e.printStackTrace();
                    OscarLogger.logException("Customer Id: " + image.getCustomerId() + "   [FAILURE]", e);
                    misUncompressedList++;
                }
                out.flush();
                out.close();
            }

        } catch (IOException i) {
            OscarLogger.logException("\nError: " + " - Can't open. ERROR... system exit", i);
            System.exit(1);
        } finally {
            try {
                if (out != null) {
                    out.flush();
                    out.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}
  1. Main
package com.mantis.signature.util;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import com.mantis.signature.model.table.SignatureImage;

public class ImageCompressUtil {
    public static String SESI_SIG_IMAGEDIR = "";
    public static String SESI_SIG_HIST_IMAGEDIR = "";
    private static String DBFID="";
    private static String DBPWD="";
    public static List ACCOUNTMAP = new ArrayList();
    public static short THREADCOUNT = 0;

    public static void main(String[] args) {
        if(args.length == 0 || args.length == 1){
            System.out.println("usage: java com.mantis.signature.util.ImageCompressUtil dirname tablename");
            return;
        }
        Runtime runt = Runtime.getRuntime();
        OscarLogger.logMessage(1,"",new Date(System.currentTimeMillis()).toString() + "  memory befor = " + runt.freeMemory());
        System.out.println(new Date(System.currentTimeMillis()).toString() + "  memory befor = " + runt.freeMemory());
        String accountFilename = "";
        try {
            Properties props = new Properties();
            props.load(ImageCompressUtil.class.getResourceAsStream("sig.properties"));
            ImageCompressUtil.DBFID = props.getProperty("DBFID");
            ImageCompressUtil.DBPWD = props.getProperty("DBPWD");
            accountFilename = props.getProperty("AccountListFile");
        } catch (IOException e) {
            e.printStackTrace();
        }
        readAccount(accountFilename);
        startDownload(ImageCompressUtil.DBFID, ImageCompressUtil.DBPWD,args[1],args[0]);
    }

    public static void readAccount(String filename){
        try {
            BufferedReader buffReader = new BufferedReader(new FileReader(new File(filename)));
            if(buffReader != null){
                String accLine = buffReader.readLine();
                while(accLine != null){
                    accLine = accLine.trim();
                    char[] accarray = accLine.toCharArray();
                    for(int i = 0 ; i < accarray.length; i++){
                        if(accarray[i] != '0'){
                            accLine = accLine.substring(i);
                            break;
                        }
                    }
                    ImageCompressUtil.ACCOUNTMAP.add(accLine);
                    accLine = buffReader.readLine();
                }
            }
            System.out.println(ImageCompressUtil.ACCOUNTMAP.size());
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void startDownload(String usrName, String passwrod, String imagedir, String tableName) {
        File f = null;
        f = new File(imagedir);
        if (!f.exists()) {
            System.out.println(imagedir + " folder is not exist, creating....");
            f.mkdirs();
        }
        ImageCompressUtil.THREADCOUNT = 32;

        ImageDumper dumper = new ImageDumper(usrName,passwrod,tableName);
        Thread t1 = new Thread(dumper);
        t1.setName("unload thread 1");
        Thread t2 = new Thread(dumper);
        t2.setName("unload thread 2");
        Thread t3 = new Thread(dumper);
        t3.setName("unload thread 3");
        Thread t4 = new Thread(dumper);
        t4.setName("unload thread 4");
        Thread t5 = new Thread(dumper);
        t5.setName("unload thread 5");
        t1.start();
        t2.start();
        t3.start();
        t4.start();
        t5.start();

        ImageWriter imageWriter = new ImageWriter(imagedir,tableName);
        Thread writer1 = new Thread(imageWriter);
        writer1.setName("writer thread 1");
        writer1.start();
    }
}
  1. Result
    Running twice with two different tables in Unix server

Producer/Consumer Model(pthread)

想用pthread做一个生产者/消费者的模型,这个事情想了很久了,一直都懒得动手,本以为很容易,写起来才知道,自己工作这几年,一直看代码,很少写代码,原来的技术功底就不怎么好,这下基本全废了~ ...
  • xiruanliuwei
  • xiruanliuwei
  • 2015年07月08日 23:39
  • 511

Java:多线程Producer-Consumer的多种实现

多线程Producer-Consumer的多种实现 更多类型Product-Consumer的实现:https://github.com/Al-assad/Produer-Consumer ...
  • Al_assad
  • Al_assad
  • 2016年11月01日 09:22
  • 643

Kafka的Producer和Consumer的示例(使用java语言)

我使用的kafka版本是:0.7.2 jdk版本是:1.6.0_20 http://kafka.apache.org/07/quickstart.html官方给的示例并不是很完整,以下代码是经过我补充...
  • qfikh
  • qfikh
  • 2017年08月24日 23:24
  • 323

Java多线程:Producer-Consumer不同的几种实现方式

生产者消费者问题是一个典型的线程同步问题。 主要有如下实现方式: wait() notifyAll() class Queue { //共享队列的目的用于...
  • langjian2012
  • langjian2012
  • 2015年03月22日 20:55
  • 1286

用python,100行完成producer与consumer模型

本程序使用python3编写,可直接运行 程序中定义了两个类,分别是producer和consumer。每个类中的run方法,用于使用者填补实际功能代码。call方法为调度使用,请不要轻易修改。 ...
  • miangangzhen
  • miangangzhen
  • 2016年03月11日 14:52
  • 1028

java--kafka编程实例--producer和consumer

先修改broker的配置文件: vim config/server.properties找到Socket Server Settings内容修改listeners的内容即可 producer代码...
  • chenxun2009
  • chenxun2009
  • 2017年05月22日 17:13
  • 393

Producer-Consumer模型:一、简介

Producer-Consumer模型是一种为了加快系统响应数据的异步模型。1.它试图解决什么问题?系统中一些慢操作(例如网络IO,数据统计等等)会阻塞主进程的运行,从而使得系统的吞吐量大大降低。如果...
  • billll
  • billll
  • 2016年10月23日 11:28
  • 587

Python producer & consumer model

第六篇(五):生产者消费者模型实现多线程异步交互 虽然标题是“生产者消费者模型实现多线程异步交互”,但这里要说的应该还包括Python的消息队列,因为这里多线程异步交互是通过Python的消息队...
  • screaming
  • screaming
  • 2016年04月02日 23:13
  • 249

Producer consumer model

生产者消费者模型 什么是生产者消费者模式   在工作中,大家可能会碰到这样一种情况:某个模块负责产生数据,这些数据由另一个模块来负责处理(此处的模块是广义的,可以是类、函数、线程、进...
  • screaming
  • screaming
  • 2016年05月11日 17:55
  • 233

kafka java代码的使用[Producer和Consumer]

用java代码对kafka消息进行消费与发送,首先我们得引入相关jar包   maven:   org.apache.kafka kafka_2.10 0.8.2.1 ...
  • u014035172
  • u014035172
  • 2017年03月29日 10:54
  • 5151
内容举报
返回顶部
收藏助手
不良信息举报
您举报文章:Java multi-threads to read DB then writting to local disk with Producer & Consumer model
举报原因:
原因补充:

(最多只允许输入30个字)