负载均衡算法实现

import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class ServerIPs {

    public static final List<String>LIST = Arrays.asList(
            "192.168.0.1",
            "192.168.0.2",
            "192.168.0.3",
            "192.168.0.4",
            "192.168.0.5",
            "192.168.0.6",
            "192.168.0.7",
            "192.168.0.8",
            "192.168.0.9",
            "192.168.0.10"
    );

    public static final Map<String,Integer> WEIGHT_LIST  = new HashMap<String,Integer>();

    static
    {
        WEIGHT_LIST.put("192.168.0.1",1);
        WEIGHT_LIST.put("192.168.0.2",8);
        WEIGHT_LIST.put("192.168.0.3",3);
        WEIGHT_LIST.put("192.168.0.4",6);
        WEIGHT_LIST.put("192.168.0.5",5);
        WEIGHT_LIST.put("192.168.0.6",5);
        WEIGHT_LIST.put("192.168.0.7",4);
        WEIGHT_LIST.put("192.168.0.8",7);
        WEIGHT_LIST.put("192.168.0.9",2);
        WEIGHT_LIST.put("192.168.0.10",9);
    }

}

随机

import java.util.Random;

public class RandomLoadBalance {

    public static String getServer()
    {
        Random random = new Random();
        int randomPos =random.nextInt(ServerIPs.LIST.size());
        return ServerIPs.LIST.get(randomPos);
    }

    public static void main(String[] args) {

        for(int i=0;i<10;i++)
        {
            System.out.println(getServer());
        }

    }

}

加权随机 (扩容)

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Random;

public class WeightRandom {

    public  static String getServer()
    {
        List<String>newServer = new ArrayList<String>();
        Random random = new Random();

        for (Map.Entry<String, Integer> map:ServerIPs.WEIGHT_LIST.entrySet())
        {
            String ip = map.getKey();
            int weight = map.getValue();
            for(int i=0;i<weight;i++)
            {
                newServer.add(ip);
            }
        }

        int pos = random.nextInt(newServer.size());
        return newServer.get(pos);
    }

    public static void main(String[] args) {

        for (int i=0;i<10;i++)
        {
            String str = getServer();
            System.out.println(String.format("ip: %s, weight: %d", str,ServerIPs.WEIGHT_LIST.get(str)));
        }
    }

}

加权随机 (区间)

import java.util.Map;
import java.util.Random;

public class WeightRandomV2 {

//    private static int totalWeight = 0;
    private int all = 0;
    public static String getServer2()
    {
        Random random = new Random();

//        System.out.println(String.format("pos: %d",pos));

        int totalWeight = 0;
//        all = totalWeight;
        for(Map.Entry<String,Integer>entry:ServerIPs.WEIGHT_LIST.entrySet())
        {
            totalWeight += entry.getValue();
        }
        int pos = random.nextInt(totalWeight);

        String res = "";
        for(Map.Entry<String,Integer>entry:ServerIPs.WEIGHT_LIST.entrySet())
        {
            int weight = entry.getValue();
            if (pos<weight)
            {
                res  = entry.getKey();
                break;
            }
            pos -= weight;
        }
        return res;
    }

    public static void main(String[] args) {

        int[]arr = new int[10];
        for (int i=0;i<20;i++)
        {
            String str = getServer2();
            int weight = ServerIPs.WEIGHT_LIST.get(str);
            arr[weight]++;
            System.out.println(String.format("ip: %s, weight: %d", str,ServerIPs.WEIGHT_LIST.get(str)));
        }
//        Arrays.sort(arr);
        for (int i=0;i<10;i++)
        {
            System.out.println(String.format("weight: %d, fre: %f", i, arr[i] / 20.0));
        }
    }
}

轮询

public class RoundRobin {

    private  static  Integer pos = 0;
    public static String getServer()
    {
        String ip = "";
        synchronized(pos)
        {
            ip = ServerIPs.LIST.get(pos);
            pos++;
            if (pos==ServerIPs.LIST.size())
            {
                pos = 0;
            }
        }
        return ip;

    }

    public static void main(String[] args) {

        for (int i = 0; i < 20; i++) {
            System.out.println(getServer());
        }
    }

}

加权轮询 (取余,区间)

import java.util.Map;
import java.util.Random;

public class RoundRobinWeight {

    private static Integer pos = 0;
    public static String getServer()
    {
        int totalWeight = 0;
        for (Map.Entry<String,Integer>entry:ServerIPs.WEIGHT_LIST.entrySet())
        {
            totalWeight += entry.getValue();
        }
        Random random = new Random();
        String ip = "";
       // System.out.println(String.format("wight: %d",totalWeight));
        synchronized (pos)
        {
            int offset = pos%totalWeight;
            //if (offset==0)offset = totalWeight;
            for (Map.Entry<String,Integer>entry:ServerIPs.WEIGHT_LIST.entrySet())
            {
                int weight = entry.getValue();
                if (offset<=weight)
                {
                    ip = entry.getKey();
                    break;
                }
                offset = offset - weight;
            }
            pos++;
        }
        return  ip;
    }

    public static void main(String[] args) {

        int[]arr = new int[10];
        for (int i=0;i<20;i++)
        {
            String str = getServer();
           // System.out.println(String.format("str: %s",str));
            int weight = ServerIPs.WEIGHT_LIST.get(str);
            arr[weight]++;
            System.out.println(String.format("ip: %s, weight: %d", str,ServerIPs.WEIGHT_LIST.get(str)));
        }
//        Arrays.sort(arr);
        for (int i=0;i<10;i++)
        {
            System.out.println(String.format("weight: %d, fre: %f", i, arr[i] / 20.0));
        }
    }

}

加权轮询,平滑

import java.util.ArrayList;
import java.util.List;

class Weight{
    private String ip;
    private Integer weight;
    private Integer currentWeight;

    public Weight(String ip, Integer weight, Integer currentWeight) {
        this.ip = ip;
        this.weight = weight;
        this.currentWeight = currentWeight;
    }

    public String getIp() {
        return ip;
    }

    public void setIp(String ip) {
        this.ip = ip;
    }

    public Integer getWeight() {
        return weight;
    }

    public void setWeight(Integer weight) {
        this.weight = weight;
    }

    public Integer getCurrentWeight() {
        return currentWeight;
    }

    public void setCurrentWeight(Integer currentWeight) {
        this.currentWeight = currentWeight;
    }
}

public class RoundRobinWeight2 {

    int totalWeight = ServerIPs.WEIGHT_LIST.values().stream().reduce(0,Integer::sum);
    List<Weight>list = new ArrayList<>();
    public String getServer()
    {
        if (list.isEmpty())
        {
            ServerIPs.WEIGHT_LIST.forEach((key,value)->list.add(new Weight(key,value,value)));
        }

        Weight maxCurrent = null;
        for (Weight weight:list)
        {
            if (maxCurrent==null || maxCurrent.getCurrentWeight()<weight.getCurrentWeight())
            {
                maxCurrent = weight;
            }
        }

        maxCurrent.setCurrentWeight(maxCurrent.getCurrentWeight() - totalWeight);
        for (Weight weight:list)
        {
            weight.setCurrentWeight(weight.getCurrentWeight() + weight.getWeight());
        }
        return maxCurrent.getIp();
    }

    public static void main(String[] args) {

        RoundRobinWeight2 r2 = new RoundRobinWeight2();
        System.out.println(String.format("weight: %d",r2.totalWeight));
        int[]arr = new int[10];
        for (int i=0;i<20;i++)
        {
            String str = r2.getServer();
            // System.out.println(String.format("str: %s",str));
            int weight = ServerIPs.WEIGHT_LIST.get(str);
            arr[weight]++;
            System.out.println(String.format("ip: %s, weight: %d", str,ServerIPs.WEIGHT_LIST.get(str)));
        }
//        Arrays.sort(arr);
        for (int i=0;i<10;i++)
        {
            System.out.println(String.format("weight: %d, fre: %f", i, arr[i] / 20.0));
        }
    }

}

一致性哈希

import java.util.SortedMap;
import java.util.TreeMap;

class Tools{
    private  int getHash(String str) {
        final int p = 16777619;
        int hash = (int) 2166136261L;
        for (int i = 0; i < str.length(); i++)
            hash = (hash ^ str.charAt(i)) * p;
        hash += hash << 13;
        hash ^= hash >> 7;
        hash += hash << 3;
        hash ^= hash >> 17;
        hash += hash << 5;
        if (hash < 0)
            hash = Math.abs(hash);
        return hash;
    }

    private TreeMap<Integer,String>virtualMap = new TreeMap<>();
    private final int VIRTUAL_NODE = 160;
    public void addNode()
    {
        for (String Ip: ServerIPs.LIST)
        {
            for (int i=0;i<VIRTUAL_NODE;i++)
            {
                int hash = getHash(Ip + i);
                virtualMap.put(hash,Ip);
            }
        }
    }

    public String getServer(String client)
    {
        int hash = getHash(client);
        SortedMap<Integer,String> subMap = virtualMap.tailMap(hash);
        Integer nodeIndex = subMap.firstKey();
        if (nodeIndex==null)
        {
            nodeIndex = virtualMap.firstKey();
        }
        return virtualMap.get(nodeIndex);
    }

}

public class ConsistentHash {


    public static void main(String[] args) {

        Tools tools = new Tools();
        tools.addNode();

        int[]arr = new int[10];
        for (int i=0;i<20;i++)
        {
            String str = tools.getServer("client" + i);
            // System.out.println(String.format("str: %s",str));
            int weight = ServerIPs.WEIGHT_LIST.get(str);
            arr[weight]++;
            System.out.println(String.format("ip: %s, weight: %d", str,ServerIPs.WEIGHT_LIST.get(str)));
        }
//        Arrays.sort(arr);
        for (int i=0;i<10;i++)
        {
            System.out.println(String.format("weight: %d, fre: %f", i, arr[i] / 10.0));
        }
    }



}

  • 0
    点赞
  • 1
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
分布式爬虫负载均衡算法实现需要考虑以下几个方面: 1. 数据分片:将待爬取的网站按照一定的规则分成多个任务,每个任务由一个爬虫节点负责爬取。这样可以避免单个节点负载过重,提高整个系统的效率。 2. 任务分配:当一个爬虫节点完成了当前任务后,需要从任务队列中获取新的任务。这个过程需要一个任务调度中心,它根据每个爬虫节点的负载情况和系统的整体负载情况,将新的任务分配给最合适的节点。 3. 动态调整:当系统的负载发生变化时,需要动态调整任务分配策略。例如,当一个节点出现故障或网络延迟时,需要将它的任务重新分配给其他节点。当系统的负载下降时,可以适当增加每个节点的任务量,提高整个系统的效率。 下面是一个简单的分布式爬虫负载均衡算法实现: 1. 数据分片:将待爬取的网站按照域名、主题或其他规则分成多个任务,每个任务由一个爬虫节点负责爬取。 2. 任务调度中心:任务调度中心维护一个任务队列,每个节点从任务队列中获取任务。任务调度中心根据每个节点的负载情况和系统的整体负载情况,将新的任务分配给最合适的节点。例如,对于一个空闲的节点,可以将多个任务分配给它;对于一个繁忙的节点,可以将少量任务分配给它。 3. 节点负载监控:每个节点需要定期向任务调度中心汇报自己的负载情况,包括已经爬取的网页数量、网络延迟、CPU、内存等指标。任务调度中心根据这些指标来判断每个节点的负载情况,并进行任务分配。 4. 动态调整:当系统的负载发生变化时,任务调度中心会根据节点负载情况和整体负载情况来动态调整任务分配策略。例如,当一个节点出现故障时,任务调度中心会将它的任务重新分配给其他节点;当系统的负载下降时,任务调度中心可以适当增加每个节点的任务量,提高整个系统的效率。 这是一个简单的分布式爬虫负载均衡算法实现,实际应用中需要根据具体情况进行优化和调整。
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值