【mock】大数据Clickhouse； Kafka； Flink； Spark 学习与性能压测数据Mock

前言

为方便大家学习大数据过程中，需要数据进行练习与调试，本次给大家分享几篇mock数据的脚本

可适用于kafka;hive;spark;flink;presto;clickhouse等等框架的操作练习

• kafka 数据mock
• hive 数据mock
• presto 数据mock
• spark 数据mock
• flink 数据mock
• clickhouse 数据mock
• doris 数据mock
• …

mock数据脚本-基础版

    private static void mock(String brokers, String topic) {
        String[] versions = new String[]{"V1.2.0", "V1.2.1", "V1.2.2"};
        String[] channels = new String[]{"小米商城", "华为商城", "AppStore"};
        String[] netTypes = new String[]{"WiFi", "4G", "5G"};
        String[] var10000 = new String[]{"cart", "browse", "pay"};
        String[] deviceTypes = new String[]{"小米11 Ultra", "iPhone 11", "iPhone 10", "iPhone 9", "华为V65"};
        Map<String, String> devices = new HashMap();
        devices.put("小米11 Ultra", "Android");
        devices.put("iPhone 11", "iOS");
        devices.put("iPhone 10", "iOS");
        devices.put("iPhone 9", "iOS");
        devices.put("华为V65", "Android");
        List<String> categories = new ArrayList();
        categories.add("Hadoop系统入门核心精讲 大数据实战");
        categories.add("SparkSQL极速入门整合Kudu实现广告业务数据分析 大数据实战");
        categories.add("Spark进阶大数据离线与实时项目实战 大数据实战");
        categories.add("学习Scala进击大数据Spark生态圈 大数据实战");
        categories.add("实战Spark3实时处理掌握两套企业级处理方案 大数据实战");
        categories.add("辣条 食品饮料");
        categories.add("矿泉水 食品饮料");
        categories.add("咖啡 食品饮料");
        categories.add("茅子 食品饮料");
        categories.add("五子 食品饮料");
        categories.add("宝马 车");
        categories.add("奔驰 车");
        categories.add("奥迪 车");
        categories.add("保时捷 车");
        categories.add("法拉利 车");
        categories.add("冲锋衣 户外");
        categories.add("帆布鞋 户外");
        categories.add("跑步鞋 户外");
        categories.add("山地车 户外");
        categories.add("运动包 户外");

        for (int i = 100; i <= 1000; ++i) {
            Access access = new Access();
            access.setUid("user_" + i);
            access.setVersion(versions[(new Random()).nextInt(versions.length)]);
            access.setDevice(UUID.randomUUID().toString());
            access.setEvent("startup");
            String deviceType = deviceTypes[(new Random()).nextInt(deviceTypes.length)];
            access.setDeviceType(deviceType);
            access.setOs((String) devices.get(deviceType));
            access.setNet(netTypes[(new Random()).nextInt(netTypes.length)]);
            String ip = getRandomIp();
            access.setIp(ip);
            access.setChannel(channels[(new Random()).nextInt(channels.length)]);
            access.setTime(System.currentTimeMillis() + 107200000000L + 86400000L);
            if (i % 3 != 0) {
                access.setNu(1);
            }
            else {
                access.setNu(0);
            }

            String info = JSON.toJSON(access).toString();
            System.out.println(info);
            Thread.sleep(1000L);
        }
    }


    private static String getRandomIp() {
        int[][] range = new int[][]{{607649792, 608174079}, {1038614528, 1039007743}, {1783627776, 1784676351}, {2035023872, 2035154943}, {2078801920, 2079064063}, {-1950089216, -1948778497}, {-1425539072, -1425014785}, {-1236271104, -1235419137}, {-770113536, -768606209}, {-569376768, -564133889}};
        Random rdint = new Random();
        int index = rdint.nextInt(10);
        String ip = num2ip(range[index][0] + (new Random()).nextInt(range[index][1] - range[index][0]));
        return ip;
    }

    private static String num2ip(int ip) {
        int[] b = new int[4];
        String x = "";
        b[0] = ip >> 24 & 255;
        b[1] = ip >> 16 & 255;
        b[2] = ip >> 8 & 255;
        b[3] = ip & 255;
        x = Integer.toString(b[0]) + "." + Integer.toString(b[1]) + "." + Integer.toString(b[2]) + "." + Integer.toString(b[3]);
        return x;
    }

mock 数据截取如下：

{"deviceType":"iPhone 10","uid":"user_1","product":{"name":"矿泉水","category":"食品饮料"},"os":"iOS","ip":"106.82.37.76","nu":1,"channel":"小米商城","time":1754026686804,"event":"cart","net":"4G","device":"4e0b3735-9f86-4b6b-a8e6-bc328a4135b8","version":"V1.2.1"}
{"deviceType":"iPhone 10","uid":"user_1","product":{"name":"实战Spark3实时处理掌握两套企业级处理方案","category":"大数据实战"},"os":"iOS","ip":"106.82.37.76","nu":1,"channel":"小米商城","time":1754026686804,"event":"browse","net":"4G","device":"4e0b3735-9f86-4b6b-a8e6-bc328a4135b8","version":"V1.2.1"}
{"deviceType":"iPhone 10","uid":"user_1","product":{"name":"宝马","category":"车"},"os":"iOS","ip":"106.82.37.76","nu":1,"channel":"小米商城","time":1754026686804,"event":"pay","net":"4G","device":"4e0b3735-9f86-4b6b-a8e6-bc328a4135b8","version":"V1.2.1"}
{"deviceType":"iPhone 10","uid":"user_1","product":{"name":"法拉利","category":"车"},"os":"iOS","ip":"106.82.37.76","nu":1,"channel":"小米商城","time":1754026686804,"event":"browse","net":"4G","device":"4e0b3735-9f86-4b6b-a8e6-bc328a4135b8","version":"V1.2.1"}

mock数据-集成到kafka

  private static KafkaProducer sendKafka(String brokers) {
        Properties properties = new Properties();
        properties.put("bootstrap.servers", brokers);
        properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        properties.put("acks", "1");
        properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        return new KafkaProducer(properties);
    }


    private static void mock(String brokers, String topic) {
        KafkaProducer producer = sendKafka("broker");

        String[] versions = new String[]{"V1.2.0", "V1.2.1", "V1.2.2"};
        String[] channels = new String[]{"小米商城", "华为商城", "AppStore"};
        String[] netTypes = new String[]{"WiFi", "4G", "5G"};
        String[] var10000 = new String[]{"cart", "browse", "pay"};
        String[] deviceTypes = new String[]{"小米11 Ultra", "iPhone 11", "iPhone 10", "iPhone 9", "华为V65"};
        Map<String, String> devices = new HashMap();
        devices.put("小米11 Ultra", "Android");
        devices.put("iPhone 11", "iOS");
        devices.put("iPhone 10", "iOS");
        devices.put("iPhone 9", "iOS");
        devices.put("华为V65", "Android");
        List<String> categories = new ArrayList();
        categories.add("Hadoop系统入门核心精讲 大数据实战");
        categories.add("SparkSQL极速入门整合Kudu实现广告业务数据分析 大数据实战");
        categories.add("Spark进阶大数据离线与实时项目实战 大数据实战");
        categories.add("学习Scala进击大数据Spark生态圈 大数据实战");
        categories.add("实战Spark3实时处理掌握两套企业级处理方案 大数据实战");
        categories.add("辣条 食品饮料");
        categories.add("矿泉水 食品饮料");
        categories.add("咖啡 食品饮料");
        categories.add("茅子 食品饮料");
        categories.add("五子 食品饮料");
        categories.add("宝马 车");
        categories.add("奔驰 车");
        categories.add("奥迪 车");
        categories.add("保时捷 车");
        categories.add("法拉利 车");
        categories.add("冲锋衣 户外");
        categories.add("帆布鞋 户外");
        categories.add("跑步鞋 户外");
        categories.add("山地车 户外");
        categories.add("运动包 户外");

        for (int i = 100; i <= 1000; ++i) {
            Access access = new Access();
            access.setUid("user_" + i);
            access.setVersion(versions[(new Random()).nextInt(versions.length)]);
            access.setDevice(UUID.randomUUID().toString());
            access.setEvent("startup");
            String deviceType = deviceTypes[(new Random()).nextInt(deviceTypes.length)];
            access.setDeviceType(deviceType);
            access.setOs((String) devices.get(deviceType));
            access.setNet(netTypes[(new Random()).nextInt(netTypes.length)]);
            String ip = getRandomIp();
            access.setIp(ip);
            access.setChannel(channels[(new Random()).nextInt(channels.length)]);
            access.setTime(System.currentTimeMillis() + 107200000000L + 86400000L);
            if (i % 3 != 0) {
                access.setNu(1);
            }
            else {
                access.setNu(0);
            }

            String info = JSON.toJSON(access).toString();
            System.out.println(info);
            producer.send(new ProducerRecord(topic, info));
            Thread.sleep(1000L);
        }
    }