井冬地址抓取

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;


import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


import com.alibaba.fastjson.JSON;


public class JD_Address {
private static int timeout = 600000;
private static int count = 0;


public static void main(String[] args) throws IOException, InterruptedException {
String province_url = "http://trade.jd.com/shopping/dynamic/consignee/getProvinces.action";
String city_url = "http://trade.jd.com/shopping/dynamic/consignee/getCitys.action?consigneeParam.provinceId=";
String county_url = "http://trade.jd.com/shopping/dynamic/consignee/getCountys.action?consigneeParam.cityId=";
String town_url = "http://trade.jd.com/shopping/dynamic/consignee/getTowns.action?consigneeParam.countyId=";


boolean province_tag = false;
boolean city_tag = false;
boolean county_tag = false;
boolean town_tag = false;



List<Address> provinces = getList(province_url, "");
for (Address province : provinces) {
if ("23".equals(province.getId())) {
province_tag = true;
}


if (province_tag) {
// 写省数据
FileUtils.writeStringToFile(new File("jdjdjd/province.txt"), province.toString(), true);
List<Address> citys = getList(city_url + province.getId(), province.getId());
// province.setChildren(citys);


for (Address city : citys) {
if ("3115".equals(city.getId())) {
city_tag = true;
}


if (city_tag) {
// 写市数据
System.out.println(city);
FileUtils.writeStringToFile(new File("jdjdjd/city.txt"), city.toString(), true);
List<Address> countys = getList(county_url + city.getId(), city.getId());
// city.setChildren(countys);


for (Address county : countys) {
if ("12751".equals(county.getId())) {
county_tag = true;
}


if (county_tag) {
// 写区或者县数据
FileUtils.writeStringToFile(new File("jdjdjd/county.txt"), county.toString(), true);
List<Address> towns = getList(town_url + county.getId(), county.getId());
// county.setChildren(towns);
for (Address town : towns) {
if ("34069".equals(town.getId())) {
town_tag = true;
}


if (town_tag) {
// 写县区数据
FileUtils.writeStringToFile(new File("jdjdjd/town.txt"), town.toString(), true);
}
}
}
}
}
}
}
}
System.out.println("=======");
// FileUtils.writeStringToFile(new File("jdjdjd/address2.js"), "var ds="
// + JSON.toJSONString(provinces));
}


private static List<Address> getList(String url, String pid) throws IOException, InterruptedException {
List<Address> list = new ArrayList<Address>();
Document doc = Jsoup.parse(new URL(url), timeout);
Elements elements = doc.select("option[value~=\\d+]");
for (Element element : elements) {
Address address = new Address();
address.setPid(pid);
address.setName(element.text().replace("*", StringUtils.EMPTY));
address.setId(element.attr("value"));
list.add(address);


// System.out.println(address);


}
++count;
if (count % 100 == 0) {
System.out.println(count);
}
if (count % 500 == 0) {
Thread.sleep(5000);
}
return list;
}


}


class Address {
private String pid;
private String name;
private String id;
private List<Address> children;


public String getPid() {
return pid;
}


public void setPid(String pid) {
this.pid = pid;
}


public String getName() {
return name;
}


public void setName(String name) {
this.name = name;
}


public List<Address> getChildren() {
return children;
}


public void setChildren(List<Address> children) {
this.children = children;
}


public String getId() {
return id;
}


public void setId(String id) {
this.id = id;
}


@Override
public String toString() {


return pid + "|" + id + "|" + name + '\n';
}


}