获取免费代理写入squid3的配置文件中

获取指定网页的免费ip代理，并且写入squid的配置文件中

官网下载直接安装即可

下载后目录

squid配置文件模板

acl SSL_ports port 443
acl Safe_ports port 80          # http
acl Safe_ports port 21          # ftp
acl Safe_ports port 443         # https
acl Safe_ports port 70          # gopher
acl Safe_ports port 210         # wais
acl Safe_ports port 1025-65535  # unregistered ports
acl Safe_ports port 280         # http-mgmt
acl Safe_ports port 488         # gss-http
acl Safe_ports port 591         # filemaker
acl Safe_ports port 777         # multiling http
acl CONNECT method CONNECT
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access deny manager
http_access allow all
http_access allow localhost
http_access deny all
http_port 6666
coredump_dir /var/spool/squid3
refresh_pattern ^ftp:           1440    20%     10080
refresh_pattern ^gopher:        1440    0%      1440
refresh_pattern -i (/cgi-bin/|\?) 0     0%      0
refresh_pattern (Release|Packages(.gz)*)$      0       20%     2880
refresh_pattern .               0       20%     4320


visible_hostname haha
via off
forwarded_for delete
#forwarded_for off
#follow_x_forwarded_for deny all
#request_header_access X-Forwarded-For deny all
#request_header_access From deny all
#request_header_access Referer deny all
#request_header_access User-Agent deny all


###cache_peer start　以下是代理ＩＰ列表

以下为代码

package main;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;


import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;


import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.Proxy;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.io.BufferedReader;
import java.io.InputStreamReader;


public class main {


public static void main(String[] args) throws InterruptedException, IOException {
// TODO Auto-generated method stub


main a = new main();
int p = 0;
List<String> iplist = new ArrayList<String>();
for (int i = 600; i < 1993; i++) {
Date day = new Date();
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
//Thread.currentThread().sleep(1000);// 毫秒
List<String> list = a.get("http://www.kuaidaili.com/free/inha/" + i + "/");
// 循环出结果放入iplist中
System.out.println(df.format(day) + "|收集ip中");
if(list.size()>0){
for (int u = 0; u < list.size(); u++)
iplist.add(list.get(u));
}
p++;
if (p == 10) {
// 去重
Set<String> set = new HashSet<String>();
List<String> newList = new ArrayList<String>();
for (String cd : iplist) {
if (set.add(cd)) {
System.out.println(cd);
newList.add(cd);
}
}
System.out.println(df.format(day) + "|更改配置文件");
a.start(newList);
p = 0;
iplist = new ArrayList<String>();
// 暂停
System.out.println(df.format(day) + "|暂停30秒");
Thread.currentThread().sleep(30000);// 毫秒
}
}


}


/*
* 处理数据
*/
public List<String>  get(String url) throws IOException {
Document doc = null;
Random ra = new Random();
doc = Jsoup.connect(url).get();
Elements links = doc.select("td[data-title]");
List<String> stringip=new ArrayList<String>();
for (int i = 0, k = 0; i < links.size(); k++) {
String ipf = links.get(i).toString();
String portF = links.get(i + 1).toString();
ipf = ipf.substring(ipf.indexOf(">") + 1, ipf.length());
ipf = ipf.substring(0, ipf.indexOf("<"));
portF = portF.substring(portF.indexOf(">") + 1, portF.length());
portF = portF.substring(0, portF.indexOf("<"));
//检测ip
int kss=createIPAddress(ipf,Integer.parseInt(portF));
if(kss==1)
{
stringip.add("cache_peer " + ipf + " parent " + portF
+ " 0 no-query weighted-round-robin weight=1 connect-fail-limit=2 allow-miss max-conn=5 name=" + ipf
+ portF);
}
i += 7;
}
return stringip;
}


public void start(List<String> list) throws InterruptedException, IOException {
// 循环15次，时间为30秒这时停止squid服务替换配置文件
Date day = new Date();
SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
//System.out.println(df.format(day) + "|开启服务squid");
//Thread.currentThread().sleep(5000);// 毫秒
// 关闭squid服务
// Runtime.getRuntime().exec("cmd /net stop squidsrv
// /user:administrator");
String peimodel = "E:/Squid/etc/squid/squidmodel.txt";// 配置文件模板位置
String pei = "E:/Squid/etc/squid/squid.conf";// 真实配置文件地址


// 获取模板内容
List<String> molist = new ArrayList<String>();
try {
String encoding = "utf-8";
File file = new File(peimodel);
if (file.isFile() && file.exists()) { // 判断文件是否存在
InputStreamReader read = new InputStreamReader(new FileInputStream(file), encoding);// 考虑到编码格式
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = null;
while ((lineTxt = bufferedReader.readLine()) != null) {
molist.add(lineTxt);
}
read.close();
} else {
System.out.println(df.format(day) + "|找不到指定的文件");
}
} catch (Exception e) {
System.out.println(df.format(day) + "|读取文件内容出错");
e.printStackTrace();
}
for (int i = 0; i < list.size(); i++)
molist.add(list.get(i).toString());
// 删除旧的配置文件
File delete = new File(pei);
if (delete.exists()) {
if (delete.delete())
System.out.println(df.format(day) + "|删除旧配置成功！");
else
System.out.println(df.format(day) + "|删除旧配置失败！");
;
} else
System.out.print(df.format(day) + "|旧配置文件不存在");


// 新建配置文件
File f = new File(pei);
if (f.exists()) {
System.out.print(df.format(day) + "|旧配置文件存在");
} else {
if (f.createNewFile())
System.out.println(df.format(day) + "|新建配置成功！");
else
System.out.println(df.format(day) + "|新建配置失败！");
}
BufferedWriter output = new BufferedWriter(new FileWriter(f));
for (int i = 0; i < molist.size(); i++)
output.write(molist.get(i).toString() + "\n");
output.write("\n" + "never_direct allow all" + "\n" + "#sky");
output.close();
System.out.println(df.format(day) + "|结束");


// 开启squid服务
// Runtime.getRuntime().exec("cmd /net start squidsrv
// /user:administrator");
// ShellExecute(0,"runas","cmd.exe","/c net user administrator
// /active:yes","","");


}


public static int createIPAddress(String ip, int port) {
URL url = null;
int k=0;
try {
url = new URL("http://www.baidu.com");
} catch (MalformedURLException e) {
System.out.println("url invalidate");
}
InetSocketAddress addr = null;
addr = new InetSocketAddress(ip, port);
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); // http proxy
InputStream in = null;
try {
URLConnection conn = url.openConnection(proxy);
conn.setConnectTimeout(100);
in = conn.getInputStream();
} catch (Exception e) {
k=0;
System.out.println("ip " + ip + " is not aviable");// 异常IP
}
String s = convertStreamToString(in);
//System.out.println(s);
// System.out.println(s);
if (s.indexOf("baidu") > 0) {// 有效IP
k=1;
System.out.println(ip + ":" + port + " is ok");
}
return k;
}


public static String convertStreamToString(InputStream is) {
if (is == null)
return "";
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
StringBuilder sb = new StringBuilder();
String line = null;
try {
while ((line = reader.readLine()) != null) {
sb.append(line + "/n");
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
is.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return sb.toString();


}
}