redis测试时插入千万数据教程

前提

redis集群6个节点，其中3个主节点，后面会向这个集群插入千万级别数据。

    nodes.add(new HostAndPort("192.168.11.180",6379));
    nodes.add(new HostAndPort("192.168.11.180",6380));
    nodes.add(new HostAndPort("192.168.11.180",6381));
    nodes.add(new HostAndPort("192.168.11.180",6389));
    nodes.add(new HostAndPort("192.168.11.180",6390));
    nodes.add(new HostAndPort("192.168.11.180",6391));

过程

造数到文件

JedisClusterPipeTest.java

package org.example;

import org.example.util.FileUtils;
import org.junit.Test;
import redis.clients.jedis.*;
import redis.clients.util.JedisClusterCRC16;

import java.util.*;

public class JedisClusterPipeTest {
    // 获取所有master节点的slot范围
    private TreeMap<Long, String> getAllMasterSlotRange(String host, int port) {
        TreeMap<Long, String> slotTree = new TreeMap<>();
        try{
            // redis带密码时
//            JedisShardInfo jedisShardInfo = new JedisShardInfo(host, port);
//            jedisShardInfo.setPassword("xxxxx");
//            Jedis jedis = new Jedis(jedisShardInfo);
            // redis不带密码时
            Jedis jedis = new Jedis(host, port);
            // 得到所有槽位： List<List<List<Object>>> list; list.sub.sub(2). get(0):get(1) 为ip和端口
            // 具体结构信息，可以打断点自己查看
            List<Object> list = jedis.clusterSlots();
            for (Object object : list) {
                List<Object> subList = (List<Object>) object;
                List<Object> master = (List<Object>) subList.get(2);
                String masterIp = new String((byte[]) master.get(0)) + ":" + master.get(1);
                long startSlot = (long) subList.get(0);
                long endSlot = (long) subList.get(1);

                slotTree.put(startSlot, masterIp);
                slotTree.put(endSlot, masterIp);
            }
            jedis.close();
        }catch(Exception e){
            e.printStackTrace();
        }
        return slotTree;
    }

    // 为每台服务器造数，key为server ip，value为造的数据集合。
    private Map<String, List<String>> getData(TreeMap<Long, String> slotHostMap, int start, int end) {
        // 为下面的key前缀分布造1000w数据
        String[] keyPres = {"key1_","key_2"};
        // key为server的ip，value为预计插入该服务器的key集合
        Map<String, List<String>> serverKeyMap = new HashMap<>();
        // key为slot id，value为该id对应的server ip
        Map<Integer, String> slotServer = new HashMap<>();
        for (String keyPre: keyPres) {
            for (int i = start; i < end; i++) {
                String key = keyPre + i;
                //获取槽号
                int slot = JedisClusterCRC16.getSlot(key);
                if (!slotServer.containsKey(slot)) {
                    slotServer.put(slot, slotHostMap.lowerEntry(Long.valueOf(slot + 1)).getValue());
                }
                String server = slotServer.get(slot);
                // 把key与server ip对应起来
                serverKeyMap.computeIfAbsent(server, serverId -> new ArrayList<>(10000)).add(key);
            }
        }
        return serverKeyMap;
    }

    @Test
    public void testBuildDataCmdToFile () {
        // 随意传入redis集群中一个节点ip，即可查到所有节点的slot分布
        TreeMap<Long, String> slotHostMap = getAllMasterSlotRange("192.168.11.180", 6379);

        System.out.println(slotHostMap.toString());
        if (slotHostMap.isEmpty()) {
            System.out.println("slot empty！");
            return;
        }

        // 一下子造数太多，占用太多内存，分批进行
        for (int i = 0; i < 1000_0000; i += 10_0000) {
            // 造数
            Map<String, List<String>> data = getData(slotHostMap, i, i + 10_0000);

            // 写入文件
            String values = " key_def_value";
            for (Map.Entry<String, List<String>> entry: data.entrySet()) {
                // 生成命令，并拼接，待写入文件
                StringBuilder sb = new StringBuilder();
                for (String key: entry.getValue()) {
                    sb.append("sadd ").append(key).append(values).append("\n");
                }
                // entry.getKey()为server ip，比如"192.168.11.180:6379"
                // 造的数据比较大的时候，超过idea的内存时，可以传入true，即可以追加的方式写入文件
                FileUtils.writeText("D:\\" + entry.getKey().replace(":", "-") + "-1000w.txt", sb.toString(), true);
            }
        }
    }

}

FileUtils

public static void writeText(String filename, String content, boolean isAppend) {
        FileOutputStream outputStream = null;
        OutputStreamWriter outputStreamWriter = null;
        BufferedWriter writer = null;
        try {
            outputStream = new FileOutputStream(filename, isAppend);
            outputStreamWriter = new OutputStreamWriter(outputStream);
            writer = new BufferedWriter(outputStreamWriter);
            writer.write(content);
        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            if(writer != null){
                try {
                    writer.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (outputStreamWriter != null){
                try {
                    outputStreamWriter.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (outputStream != null){
                try {
                    outputStream.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

每个节点pipe到redis

上面程序执行后，可以看到D盘生成3个文件，总共数据量2000w，即set结构的key2000w个。

上传到centos服务器/opt，执行：

cat /opt/192.168.11.180-6379-1000w.txt | redis-cli -h 192.168.11.180 -p 6379 --pipe
cat /opt/192.168.11.180-6380-1000w.txt | redis-cli -h 192.168.11.180 -p 6380 --pipe
cat /opt/192.168.11.180-6381-1000w.txt | redis-cli -h 192.168.11.180 -p 6381 --pipe

若服务器带密码，可以下面这样

cat /opt/192.168.11.180-6379-1000w.txt | redis-cli -h 192.168.11.180 -p 6379 -a password --pipe

–pipe无法应用于集群，但可以应用于集群的单台机器