Hive+Sqoop+Mysql整合代码

最新推荐文章于 2023-05-15 10:07:01 发布
阿誠的数据杂货铺
最新推荐文章于 2023-05-15 10:07:01 发布
阅读量156
点赞数
分类专栏： java
java 专栏收录该内容
21 篇文章 0 订阅
订阅专栏
1.模拟数据代码
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Serializable;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

/**
 * 产生测试数据：
 * 数据format：
 * 记录时间                 车牌号码            车速                     道路编号       监控地点        摄像头编号
 * date_time     vehicle_plate     vehicle_speed    road_id   monitor_id    camera_id
 * 
 * 中间使用'\t'隔开
 * 16/01/2019 10:20:30 SCN89000J 124 10002 20004 40007
 * 
 * 具体说明：
 * 道路编号
 * 10001 - 10100
 * 
 * 监控地点 - 在一条道路上面有2个监控点
 * 20001 - 20200
 * 
 * 摄像头编号 - 在一个监控点上面2个摄像头
 * 40001 - 40400
 * 
 * 道路：      10001                         10002
 * 监控：      20001-20002                   20003-20004
 * 摄像头：  40001-40002-40003-40004       40005-40006-40007-40008
 * 
 * 车速： 1-300。 如果大于260，则为超速行驶
 * 
 * 车牌： SCN89000J
 * 
 * 记录时间： 16/01/2019 10:20:30 
 * 
 * @author Hongten
 */
public class DataGenerate {

    Integer[] roadIdArray = new Integer[Common.ROAD_NUM];

    public static void main(String[] args) {
        long begin = System.currentTimeMillis();
        DataGenerate dataGenerate = new DataGenerate();
        dataGenerate.init();
        dataGenerate.generateData();
        long end = System.currentTimeMillis();
        System.out.println("Total: " + (end - begin) + " ms");
    }

    public void init() {
        // create files
        FileUtils.createFile(Common.VEHICLE_LOG);
        FileUtils.createFile(Common.ROAD_MONITOR_CAMERA_RELATIONSHIP);

        generateRoadIds();
    }

    /**
     * 道路： 10001 10002 监控： 20001-20002 20003-20004 摄像头： 40001-40002-40003-40004
     * 40005-40006-40007-40008
     */
    public void generateRoadIds() {
        StringBuilder readMonitorCameraRelationship = new StringBuilder();
        for (int i = 0; i < Common.ROAD_NUM; i++) {
            int roadId = 10000 + (i + 1);// 10001
            roadIdArray[i] = roadId;

            int monitorB = roadId * 2;// 20002
            int monitorA = monitorB - 1;// 20001

            int cameraAB = monitorA * 2;// 40002
            int cameraAA = cameraAB - 1;// 40001

            int cameraBB = monitorB * 2;// 40004
            int cameraBA = cameraBB - 1;// 40003

            // monitorA
            // 10001 20001 40001
            // 10001 20001 40002
            readMonitorCameraRelationship.append(roadId).append(Common.SEPARATOR).append(monitorA).append(Common.SEPARATOR).append(cameraAA).append(Common.LINE_BREAK);
            readMonitorCameraRelationship.append(roadId).append(Common.SEPARATOR).append(monitorA).append(Common.SEPARATOR).append(cameraAB).append(Common.LINE_BREAK);
            // monitorB
            // 10001 20002 40003
            // 10001 20002 40004
            readMonitorCameraRelationship.append(roadId).append(Common.SEPARATOR).append(monitorB).append(Common.SEPARATOR).append(cameraBA).append(Common.LINE_BREAK);
            readMonitorCameraRelationship.append(roadId).append(Common.SEPARATOR).append(monitorB).append(Common.SEPARATOR).append(cameraBB).append(Common.LINE_BREAK);
        }
        saveData(Common.ROAD_MONITOR_CAMERA_RELATIONSHIP, readMonitorCameraRelationship.toString());
    }

    public void saveData(String pathFileName, String newContent) {
        //remove the last '\n'
        newContent = newContent.substring(0, newContent.length() - 1);
        FileUtils.saveFile(pathFileName, newContent);
    }

    public void generateData() {
        //StringBuffer可以保证线程安全
        StringBuffer contentSb = new StringBuffer();
        SimpleDateFormat simpleDateFormat_ddMMyyyy = new SimpleDateFormat(Common.DATE_FORMAT_YYYYMMDD);
        Date today = new Date();
        String date = simpleDateFormat_ddMMyyyy.format(today);
        Random random = new Random();
        
        //异常道路
        List<Integer> errorRoadIdList = new ArrayList<Integer>();
        generateErrorRoadIdList(random, errorRoadIdList);
                
        long begin = System.currentTimeMillis();
        
        //使用多线程
        ExecutorService exec = Executors.newCachedThreadPool();
        for (int i = 0; i < Common.VEHICLE_NUMBER; i++) {
            String vehiclePlate = VehiclePlateGenerateSG.generatePlate();
            //使用Future和Callable组合，可以获取到返回值
            Future<String> result = exec.submit(new GenerateVehicleLog(date, random, errorRoadIdList, vehiclePlate, roadIdArray));
            try {
                contentSb.append(result.get());
                if(i % 100 == 0){
                    System.out.println(i);
                }
                
                if(i != 0 && i % 900 == 0){
                    long end = System.currentTimeMillis();
                    System.out.println(i + " sleeping 1 seconds." + " " + (end - begin)/1000 + " s");
                    //waiting the pre-task to finish.
                    TimeUnit.SECONDS.sleep(1);
                }
            } catch (InterruptedException e) {
                e.printStackTrace();
            } catch (ExecutionException e) {
                e.printStackTrace();
            }
            //System.out.println(contentSb.toString());
        }
        exec.shutdown();
        saveData(Common.VEHICLE_LOG, contentSb.toString());
    }

    private void generateErrorRoadIdList(Random random, List<Integer> errorRoadIdList) {
        for (int x = 0; x < Common.ROAD_ERROR_NUM; x++) {
            if (errorRoadIdList.contains(roadIdArray[random.nextInt(roadIdArray.length)])) {
                generateErrorRoadIdList(random, errorRoadIdList);
            } else {
                errorRoadIdList.add(roadIdArray[random.nextInt(roadIdArray.length)]);
            }
        }
    }
}

class GenerateVehicleLog implements Callable<String>{

    StringBuffer contentSb;
    String date;
    Random random;
    List<Integer> errorRoadIdList;
    String vehiclePlate;
    Integer[] roadIdArray;
    
    public GenerateVehicleLog( String date, Random random, List<Integer> errorRoadIdList, String vehiclePlate, Integer[] roadIdArray){
        this.contentSb = new StringBuffer();
        this.date = date;
        this.random = random;
        this.errorRoadIdList = errorRoadIdList;
        this.vehiclePlate = vehiclePlate;
        this.roadIdArray = roadIdArray;
    }
    
    @Override
    public String call() throws Exception {
        return getVehicleLog(contentSb, date, random, errorRoadIdList, vehiclePlate, roadIdArray);
    }
    
    private String getVehicleLog(StringBuffer contentSb, String date, Random random, List<Integer> errorRoadIdList, String vehiclePlate, Integer[] roadIdArray) {
        // 每一辆车产生记录在100条记录以内
        // 即最多过100个监控点
        // 即最多过50条路
        // 这里可以根据需要调节
        for (int n = 0; n < random.nextInt(Common.ROAD_NUM); n++) {
            int roadId = roadIdArray[random.nextInt(roadIdArray.length)];
            Integer[] monitorIdArray = new Integer[2];
            Integer[] cameraIdArray = new Integer[2];
            boolean isAllError = false;
            int monitorId = 0;
            if (errorRoadIdList.contains(roadId)) {
                // System.out.println("find error road.... " + roadId +
                // " for vehicle : " + vehiclePlate);
                if (roadId % 2 == 0) {
                    // 监控设备全部坏掉
                    isAllError = true;
                } else {
                    // 部分坏掉
                    monitorIdArray[0] = roadId * 2 - 1;
                    monitorIdArray[1] = roadId * 2 - 1;

                    monitorId = monitorIdArray[random.nextInt(monitorIdArray.length)];

                    cameraIdArray[0] = roadId * 4 - 3;
                    cameraIdArray[1] = roadId * 4 - 2;
                }
            } else {
                monitorIdArray[0] = roadId * 2 - 1;
                monitorIdArray[1] = roadId * 2;

                monitorId = monitorIdArray[random.nextInt(monitorIdArray.length)];

                cameraIdArray[0] = monitorId * 2 - 1;
                cameraIdArray[1] = monitorId * 2;
            }

            if (!isAllError) {
                // 16/01/2019 10:20:30 SCN89000J 124 10002 20004 40007
                contentSb.append(date).append(Common.BLANK).append(StringUtils.fulfuill(String.valueOf(random.nextInt(25)))).append(Common.COLON).append(StringUtils.fulfuill(String.valueOf(random.nextInt(61)))).append(Common.COLON).append(StringUtils.fulfuill(String.valueOf(random.nextInt(61)))).append(Common.SEPARATOR);
                contentSb.append(vehiclePlate).append(Common.SEPARATOR);
                contentSb.append((random.nextInt(Common.MAX_SPEED) + 1)).append(Common.SEPARATOR);
                contentSb.append(roadId).append(Common.SEPARATOR);
                contentSb.append(monitorId).append(Common.SEPARATOR);
                contentSb.append(cameraIdArray[random.nextInt(cameraIdArray.length)]).append(Common.LINE_BREAK);
            }
        }
        return contentSb.toString();
    }
}



class FileUtils implements Serializable {

    private static final long serialVersionUID = 1L;

    public static boolean createFile(String pathFileName) {
        try {
            File file = new File(pathFileName);
            if (file.exists()) {
                System.err.println("Find file" + pathFileName + ", system will delete it now!!!");
                file.delete();
            }
            boolean createNewFile = file.createNewFile();
            System.err.println("create file " + pathFileName + " success!");
            return createNewFile;
        } catch (IOException e) {
            e.printStackTrace();
        }
        return false;
    }

    public static void saveFile(String pathFileName, String newContent) {
        FileOutputStream fos = null;
        OutputStreamWriter osw = null;
        PrintWriter pw = null;
        try {
            String content = newContent;
            File file = new File(pathFileName);
            fos = new FileOutputStream(file, true);
            osw = new OutputStreamWriter(fos, Common.CHARSETNAME_UTF_8);
            pw = new PrintWriter(osw);
            pw.write(content);
            pw.close();
            osw.close();
            fos.close();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (pw != null) {
                pw.close();
            }
            if (osw != null) {
                try {
                    osw.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (fos != null) {
                try {
                    fos.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
}

class StringUtils {

    public static String fulfuill(String str) {
        if (str.length() == 1) {
            return Common.ZERO + str;
        }
        return str;
    }
}

/**
 * From wiki:
 * https://en.wikipedia.org/wiki/Vehicle_registration_plates_of_Singapore<br>
 * 
 * A typical vehicle registration number comes in the format "SKV 6201 B":<br>
 * 
 * <li>S – Vehicle class ("S", with some exceptions, stands for a private
 * vehicle since 1984)</li><br>
 * <li>KV – Alphabetical series ("I" and "O" are not used to avoid confusion
 * with "1" and "0")</li><br>
 * <li>6201 – Numerical series</li><br>
 * <li>B – Checksum letter ("F","I", "N", "O", "Q", "V" and "W" are never used
 * as checksum letters; absent on special government vehicle plates and events
 * vehicle plates)</li>
 * 
 */
class VehiclePlateGenerateSG implements Serializable {

    private static final long serialVersionUID = -8006144823705880339L;

    public static Random random = new Random();

    // 主要目的就是使得产生的数字字符不在这个list里面
    // 如果在这个list里面找到，那么需要重新产生
    private static List<String> uniqueList = new ArrayList<String>();

    public static String generatePlate() {
        // System.out.println(ALPHABETICAL_ARRAY[18]);// S
        String alphabeticalSeries = getAlphabeticalStr(Common.ALPHABETICAL_ARRAY, random) + getAlphabeticalStr(Common.ALPHABETICAL_ARRAY, random);
        // System.out.println(alphabeticalSeries);//KV

        String numbericalSeries = getNumericalSeriesStr(random);
        // System.out.println(numbericalSeries);//62010

        String checksumLetter = getChecksumLetterStr(Common.ALPHABETICAL_ARRAY, random);
        // System.out.println(checksumLetter);//B

        String singaporeVehiclePlate = Common.ALPHABETICAL_ARRAY[18] + alphabeticalSeries + numbericalSeries + checksumLetter;
        return singaporeVehiclePlate;
    }

    private static String getAlphabeticalStr(String[] ALPHABETICAL_ARRAY, Random random) {
        String alphabeticalStr = Common.ALPHABETICAL_ARRAY[random.nextInt(Common.ALPHABETICAL_ARRAY.length)];
        // "I", "O"
        if (!(alphabeticalStr.equals(Common.ALPHABETICAL_ARRAY[8]) || alphabeticalStr.equals(Common.ALPHABETICAL_ARRAY[14]))) {
            return alphabeticalStr;
        } else {
            return getAlphabeticalStr(Common.ALPHABETICAL_ARRAY, random);
        }
    }

    private static String getNumericalSeriesStr(Random random) {
        // 为了区别真实的车牌，我们把数字设置为5位
        String numericalStr = random.nextInt(10) + "" + random.nextInt(10) + "" + random.nextInt(10) + "" + random.nextInt(10) + "" + random.nextInt(10);
        if (uniqueList.contains(numericalStr)) {
            // 如果存在，则重新产生
            return getNumericalSeriesStr(random);
        } else {
            uniqueList.add(numericalStr);
            return numericalStr;
        }
    }

    private static String getChecksumLetterStr(String[] ALPHABETICAL_ARRAY, Random random) {
        String checksumLetter = ALPHABETICAL_ARRAY[random.nextInt(ALPHABETICAL_ARRAY.length)];
        // "F","I", "N", "O", "Q", "V" and "W"
        if (!(checksumLetter.equals(Common.ALPHABETICAL_ARRAY[5]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[8]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[13]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[14]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[16]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[21]) || checksumLetter.equals(Common.ALPHABETICAL_ARRAY[22]))) {
            return checksumLetter;
        } else {
            return getChecksumLetterStr(ALPHABETICAL_ARRAY, random);
        }
    }
}

class Common implements Serializable {
    private static final long serialVersionUID = 1L;

    public static String VEHICLE_LOG = "./vehicle_log";
    public static String ROAD_MONITOR_CAMERA_RELATIONSHIP = "./road_monitor_camera_relationship";

    public static final String[] ALPHABETICAL_ARRAY = new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" };
    public static final String DATE_FORMAT_YYYYMMDD = "dd/MM/yyyy";
    public static final String CHARSETNAME_UTF_8 = "UTF-8";
    public static final String SEPARATOR = "\t";
    public static final String LINE_BREAK = "\n";
    public static final String BLANK = " ";
    public static final String COLON = ":";
    public static final String ZERO = "0";
    
    //车辆数
    public static final int VEHICLE_NUMBER = 100000;
    //道路数
    public static final int ROAD_NUM = 400;
    public static final int ROAD_ERROR_NUM = 8;
    //最大车速
    public static final int MAX_SPEED = 300;
}