java利用Jsoup爬取网上免费代理IP，搭建自己的Ip池

package com.example.demo.util;

import com.beust.jcommander.internal.Lists;

import org.apache.commons.collections.CollectionUtils;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.net.URL;
import java.util.List;


public class FetchProxyIpUtil {



        public static List<String>  getProxyIp() throws IOException {
                List<String> proxyIpList = Lists.newArrayList();
                //解析网页（jsoup返回document就是浏览器document对象）
                Document document = Jsoup.parse(new URL("http://www.xiladaili.com/gaoni/"), 60000);
                Element body = document.selectFirst(".fl-table > tbody:nth-child(2)");
                Elements tr = body.getElementsByTag("tr");
                for(Element r : tr){
                        Elements td = r.getElementsByTag("td");
                        if(CollectionUtils.isNotEmpty(td)){
                                Element element = td.get(0);
                                String proxyIp = element.text();
                                proxyIpList.add(proxyIp);
                        }
                }
                return proxyIpList;
        }

        public static void main(String[] args) throws IOException {

                System.out.println(getProxyIp());
        }

}

 

返回结果：

[154.127.79.199:8080, 183.87.153.98:49602, 223.243.245.45:9999, 103.111.55.210:47623, 115.219.1.184:3256, 110.18.155.130:9999, 112.195.241.132:3256, 110.18.155.199:9999, 104.254.238.122:42415, 181.78.18.25:999, 183.148.226.161:3256, 139.224.211.212:8080, 124.112.171.132:9999, 103.253.113.54:443, 91.203.36.102:45551, 112.195.243.211:3256, 37.26.86.206:47464, 47.98.170.216:8080, 113.194.49.162:9999, 176.235.182.99:8080, 39.71.123.83:8118, 200.199.90.218:3128, 117.65.1.218:3256, 115.219.2.87:3256, 59.110.213.236:8118, 218.91.1.102:9999, 117.31.45.39:8888, 106.14.41.100:8080, 45.130.229.230:443, 45.64.11.1:8080, 157.230.103.91:37107, 112.195.241.196:3256, 112.195.240.117:3256, 104.254.238.122:8217, 58.176.147.14:8193, 136.228.128.236:8080, 182.253.107.210:8080, 36.91.108.138:3128, 112.195.240.231:3256, 131.161.237.113:8090, 202.150.138.234:8080, 90.189.110.170:3128, 202.61.99.26:3128, 195.53.49.11:3128, 103.83.178.166:8181, 103.135.225.10:3128, 187.243.253.2:8080, 175.110.211.166:8080, 116.68.170.190:8080, 157.100.53.108:999]