maven依赖:
<dependencies>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>4.5.15</version>
</dependency>
<dependency>
<groupId>cn.wanghaomiao</groupId>
<artifactId>JsoupXpath</artifactId>
<version>0.1.1</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.8</version>
</dependency>
<dependency>
<groupId>com.gitee.jd-platform-opensource</groupId>
<artifactId>asyncTool</artifactId>
<version>37d5425b6f</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
<optional>true</optional>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.2</version>
<optional>true</optional>
</dependency>
</dependencies>
<repositories>
<repository>
<id>jitpack.io</id>
<url>https://jitpack.io</url>
</repository>
</repositories>
package com.xiaoyun;
import cn.hutool.core.map.MapUtil;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import cn.wanghaomiao.xpath.exception.NoSuchAxisException;
import cn.wanghaomiao.xpath.exception.NoSuchFunctionException;
import cn.wanghaomiao.xpath.exception.XpathSyntaxErrorException;
import cn.wanghaomiao.xpath.model.JXDocument;
import com.jd.platform.async.callback.ICallback;
import com.jd.platform.async.callback.IWorker;
import com.jd.platform.async.executor.Async;
import com.jd.platform.async.executor.timer.SystemClock;
import com.jd.platform.async.worker.WorkResult;
import com.jd.platform.async.wrapper.WorkerWrapper;
import lombok.Builder;
import lombok.Data;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class ProxyUtil {
private static Map<String, Map<String, String>> cacheBodyProxy = new HashMap();
private static AtomicInteger num = new AtomicInteger(0);
public static void main(String[] args) {
long now = SystemClock.now();
System.out.println("begin-" + now);
List<ProxyBean> okList = okAbroadProxyIp();
List<ProxyBean> okList001 = okDomesticProxyIp();
Async.shutDown();
System.out.println("end-" + SystemClock.now());
System.err.println("cost-" + (SystemClock.now() - now));
System.out.println(okList001);
System.out.println(okList);
System.out.println(okList001.size());
System.out.println(okList.size());
System.out.println(num);
}
private static List<ProxyBean> okAbroadProxyIp() {
List<ProxyBean> list = getAbroadProxyIp();
List<ProxyBean> okList = checkProxy(list);
return okList;
}
private static List<ProxyBean> okDomesticProxyIp() {
List<ProxyBean> list = getDomesticProxyIp();
List<ProxyBean> okList = checkProxy(list);
return okList;
}
private static List<ProxyBean> checkProxy(List<ProxyBean> list) {
List<ProxyBean> ok = list.stream().parallel().map(item -> {
try {
return executeWorkFunction(item);
} catch (Exception e) {
return null;
}
}).filter(Objects::nonNull).collect(Collectors.toList());
return ok;
}
private static ProxyBean executeWork(ProxyBean item) {
HttpUtil.createGet("www.baidu.com")
.setProxy(getProxy(item))
.timeout(5000)
.execute()
.body();
return item;
}
private static ProxyBean executeWorkFunction(ProxyBean item) throws Exception {
WorkFunction<ProxyBean, ProxyBean> wf = new WorkFunction<ProxyBean, ProxyBean>() {
@Override
public ProxyBean action(ProxyBean object, Map<String, WorkerWrapper> allWrappers) {
HttpResponse execute = HttpUtil.createGet("www.baidu.com")
.setProxy(getProxy(item))
.timeout(5000)
.execute();
boolean isOk = execute.isOk();
System.out.println(Thread.currentThread().getName() + "----------------------");
if (isOk) {
num.getAndIncrement();
return object;
} else {
return null;
}
}
};
WorkerWrapper<ProxyBean, ProxyBean> build = new WorkerWrapper.Builder<ProxyBean, ProxyBean>()
.worker(wf)
.callback(wf)
.param(item)
.build();
Async.beginWork(500000, build);
return build.getWorkResult().getResult();
}
private static List<ProxyBean> getAbroadProxyIp() {
return getIpBeans(1);
}
private static List<ProxyBean> getDomesticProxyIp() {
return getIpBeans(0);
}
private static List<ProxyBean> getIpBeans(Integer index) {
String url = "http://www.xsdaili.cn";
String xpathUrl = "//div[@class='cont']/a/@href";
String xpath = "//div[@class='cont']/text()";
List<Object> listUrl = getBodyForXpath(url, xpathUrl);
String urlHtml = url + listUrl.get(index).toString();
List<Object> list = getBodyForXpath(urlHtml, xpath);
Pattern p = Pattern.compile("(\\d+\\.\\d+\\.\\d+\\.\\d+)\\:(\\d+)");
Matcher m = p.matcher(list.get(0).toString());
List<ProxyBean> ipBeans = new ArrayList<>();
while (m.find()) {
ProxyBean bean = ProxyBean.builder().ip(m.group(1)).port(Integer.parseInt(m.group(2))).build();
ipBeans.add(bean);
}
return ipBeans;
}
private static List<Object> getBodyForXpath(String url, String xpathName) {
String body = getBody(url);
List<Object> objects = selForXpath(body, xpathName);
return objects;
}
private static List<Object> proxyGetBodyForXpath(String url, String xpathName, ProxyBean proxyBean) {
String body = proxyGetBody(proxyBean, url);
List<Object> objects = selForXpath(body, xpathName);
return objects;
}
private static String getBody(String url) {
Map<String, String> map = cacheBodyProxy.get(url);
if (!MapUtil.isEmpty(map)) {
String body = map.get("static");
if (!isEmpty(body)) {
return body;
}
}
String body = HttpUtil.createGet(url)
.timeout(5000)
.execute().body();
Map<String, String> staticMap = new HashMap<>();
staticMap.put("static", body);
cacheBodyProxy.put(url, staticMap);
return body;
}
private static String proxyGetBody(ProxyBean proxyBean, String url) {
if (proxyBean != null) {
Map<String, String> map = cacheBodyProxy.get(url);
if (!MapUtil.isEmpty(map)) {
String bodyCache = map.get(proxyBean.getIp());
if (!isEmpty(bodyCache)) {
return bodyCache;
}
}
String ip = proxyBean.getIp();
Proxy proxy = getProxy(proxyBean);
String body = HttpUtil.createGet(url)
.setProxy(proxy)
.timeout(5000)
.execute().body();
Map<String, String> proxyMap = new HashMap<>();
proxyMap.put(ip, body);
cacheBodyProxy.put(url, proxyMap);
return body;
}
return null;
}
private static Proxy getProxy(ProxyBean proxyBean) {
return new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyBean.getIp(), proxyBean.getPort()));
}
public static boolean isEmpty(String str) {
return str == null || str.length() == 0;
}
private static List<Object> selForXpath(String body, String xpathName) {
JXDocument jxDocument = new JXDocument(body);
List<Object> sel = new ArrayList<>();
try {
sel = jxDocument.sel(xpathName);
} catch (NoSuchAxisException e) {
e.printStackTrace();
} catch (NoSuchFunctionException e) {
e.printStackTrace();
} catch (XpathSyntaxErrorException e) {
e.printStackTrace();
}
return sel;
}
}
class WorkFunction<T, V> implements IWorker<T, V>, ICallback<T, V> {
@Override
public void begin() {
}
@Override
public void result(boolean success, T param, WorkResult<V> workResult) {
}
@Override
public V action(T object, Map<String, WorkerWrapper> allWrappers) {
return null;
}
@Override
public V defaultValue() {
return null;
}
}
@Data
@Builder
class ProxyBean {
private String ip;
private Integer port;
}