学习小结,自己以后回顾使用,有错误的欢迎补充指正。
学习爬虫第二天笔记,代码。最主要的是代码和pom文件的配置
主体JAVA代码:
package co.itheima.sprider;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Map;
import org.apache.http.ParseException;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.google.gson.Gson;
public class JdSprider {
public static void main(String[] args) throws Exception {
spider();
dopaging();
}
private static void dopaging() throws Exception{
int page = 1;
while (page < 1000) {
String url = "https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA&cid2=653&cid3=655&page="
+ (2 * page - 1);
System.out.println(url);
CloseableHttpResponse response = getHtml(url);
product(response);
page++;
}
}
private static void spider() throws Exception {
String indexurl = "https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&wq=%E6%89%8B%E6%9C%BA&pvid=4bef23fee3e34c12b1826f555400c894";
CloseableHttpResponse response = getHtml(indexurl);
product(response);
}
private static void product(CloseableHttpResponse response) throws IOException {
if (200 == response.getStatusLine().getStatusCode()) {
String entity = EntityUtils.toString(response.getEntity(), Charset.forName("utf-8"));
Document doc = Jsoup.parse(entity);
Elements elements = doc.select("#J_goodsList li[data-pid]");
for (Element element : elements) {
try {
String pId = element.attr("data-pid");
paramenter(pId);
} catch (Exception e) {
System.out.println("访问失败!" + element.attr("data-pid") + e);
}
}
}
}
private static CloseableHttpResponse getHtml(String indexurl) throws IOException, ClientProtocolException {
HttpGet httpGet = new HttpGet(indexurl);
CloseableHttpClient httpClient = HttpClients.createDefault();
CloseableHttpResponse response = httpClient.execute(httpGet);
return response;
}
private static void paramenter(String pId) throws Exception {
String pUrl = "https://item.jd.com/" + pId + ".html";
CloseableHttpResponse detailRes = getHtml(pUrl);
if (200 == detailRes.getStatusLine().getStatusCode()) {
String html = EntityUtils.toString(detailRes.getEntity(), Charset.forName("utf-8"));
Document document = Jsoup.parse(html);
Product product = getproduct(document);
product.setId(pId);
product.setUrl(pUrl);
String priceUrl = "https://p.3.cn/prices/mgets?skuIds=J_" + pId;
CloseableHttpResponse priceRes = getHtml(priceUrl);
if (200 == priceRes.getStatusLine().getStatusCode()) {
String priceJson = EntityUtils.toString(priceRes.getEntity(), Charset.forName("utf-8"));
Gson gson = new Gson();
ArrayList<Map> resultList = gson.fromJson(priceJson, ArrayList.class);
Map<String, String> map = (Map<String, String>) resultList.get(0);
String price = map.get("op");
product.setPrice(price);
}
System.out.println(product);
}
}
private static Product getproduct(Document document) {
Product product = new Product();
String name = document.select("[class=parameter2 p-parameter-list] li").get(0).text();
product.setName(name);
String title = document.select(".sku-name").get(0).text();
product.setTitle(title);
return product;
}
}
创建Product对象用于接收获取到的手机对象的数据,输出到控制台方便我们监测代码出现的错误。
注意要先创建GET对象,将GET对象放到连接方法中,打开连接,获取连接的响应对象
String entity = EntityUtils.toString(response.getEntity(), Charset.forName("utf-8"));
Document doc = Jsoup.parse(entity)
从响应中拿到响应体对象,设置编码为utf-8,转换成Document对象,通过Document选择器获得我们想要的elements对象集合,遍历集合获取对象中我们需要的属性。封装到Product中,用tostring方法输出到控制台。
注意其中的各种选择器的方法和名字不能写错,造成拿不到属性值的问题。
public static void main(String[] args) {
final ArrayDeque<String> arrayDeque = new ArrayDeque<String>();
for (int i = 0; i <= 99; i++) {
arrayDeque.offer("element" + i);
//test1();
}
new Thread(new Runnable() {
public void run() {
while (true) {
String element = arrayDeque.poll();
System.out.println(Thread.currentThread().getId() +" "+element);
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}).start();
new Thread(new Runnable() {
public void run() {
while (true) {
String element = arrayDeque.poll();
System.out.println(Thread.currentThread().getId() +" "+element);
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}).start();
}
private static void test1() {
ArrayDeque<String> arrayDeque = new ArrayDeque<String>();
arrayDeque.offer("element01");
arrayDeque.offer("element02");
arrayDeque.offer("element03");
arrayDeque.offer("element04");
String element = arrayDeque.poll();
System.out.println(element);
element = arrayDeque.poll();
System.out.println(element);
for (String string : arrayDeque) {
System.out.println(string);
}
}
下午学习的队列的一点知识,上面是已经理解的一部分内容,队列的特性是先进先出,拿出来的数据就不存在于队列中了。
还有多线程问题,多线程参与爬虫,队列存放数据放进数据库中的问题,暂时没搞懂。