java爬虫_java爬虫 案例

packagecom.zjazn;importcom.sun.org.apache.bcel.internal.generic.RETURN;importcom.sun.xml.internal.ws.api.server.InstanceResolver;importorg.apache.http.HttpEntity;importorg.apache.http.client.methods.CloseableHttpResponse;importorg.apache.http.client.methods.HttpGet;importorg.apache.http.impl.client.CloseableHttpClient;importorg.apache.http.impl.client.HttpClientBuilder;importorg.apache.http.impl.client.HttpClients;importorg.apache.http.util.EntityUtils;importorg.jsoup.Jsoup;importorg.jsoup.nodes.Document;importorg.jsoup.nodes.Element;importorg.jsoup.select.Elements;importsun.net.www.http.HttpClient;importjava.io.File;importjava.io.FileOutputStream;importjava.io.IOException;importjava.io.InputStream;importjava.net.MalformedURLException;importjava.net.URL;importjava.net.URLConnection;importjava.util.ArrayList;importjava.util.List;public classData {

//2、解析数据public static voidmain(String[] args) {

String html=getData();

Document htmledThisDocument=Jsoup.parse(html);

List myData=new ArrayList();

Elements courses= htmledThisDocument.select(".learn-path-container>div");for(Element course:courses){

String courseName= course.select("a>div").first().text();

String courseNum= course.select("a>div").last().text();if(courseNum.indexOf("门")>-1){int num = Integer.parseInt(courseNum.substring(0, courseNum.indexOf("门")));

String imgPath= course.select("a>img").attr("src");

String fuffix= imgPath.substring(imgPath.lastIndexOf("."));

MyData myData6= newMyData();

myData6.setName(courseName);

myData6.setImgPath(imgPath);

myData6.setNum(num);

myData.add(myData6);

downloadFile(imgPath,"E://myimg",courseName+fuffix);

}

}

System.out.println(myData.toString());

}

//1、获取数据(html)public staticString getData(){

CloseableHttpClient httpClient=HttpClients.createDefault();

HttpGet httpGet= new HttpGet("https://www.lanqiao.cn/paths/");

CloseableHttpResponse response=null;

HttpEntity entity=null;

String html=null;try{

response= httpClient.execute(httpGet);//发送请求

if(response.getStatusLine().getStatusCode() ==200){

entity= response.getEntity();//获取html

html= EntityUtils.toString(entity,"UTF-8");//用指定编码解析html

}returnhtml;

}catch(IOException e) {

e.printStackTrace();

}return null;

}

//传入资源链接，下载资源的方法，比如下载图片