Java(爬虫)底层实现爬取京东spu+sku数据

第一: 加入pom指定坐标(采用springboot的定时任务进行爬取数据)

<parent>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-parent</artifactId>
		<version>2.0.5.RELEASE</version>
		<relativePath/> <!-- lookup parent from repository -->
	</parent>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
		<java.version>1.8</java.version>
	</properties>

	<dependencies>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-jpa</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
		</dependency>

		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<scope>runtime</scope>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>


		</dependency>

		<!-- HttpClient -->
		<dependency>
			<groupId>org.apache.httpcomponents</groupId>
			<artifactId>httpclient</artifactId>
		</dependency>

		<!--Jsoup-->
		<dependency>
			<groupId>org.jsoup</groupId>
			<artifactId>jsoup</artifactId>
			<version>1.10.3</version>
		</dependency>

		<!--工具包-->
		<dependency>
			<groupId>org.apache.commons</groupId>
			<artifactId>commons-lang3</artifactId>
		</dependency>
	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.springframework.boot</groupId>
				<artifactId>spring-boot-maven-plugin</artifactId>
			</plugin>
		</plugins>
	</build>

第二步(2.1) 指定指定获取的数据库表以及对应的pojo

package com.jd.crawler.pojo;

import javax.persistence.*;

@Entity
@Table(name = "jd_item")
public class JdItem {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
  private Long id;
  private Long spu;
  private Long sku;
  private String title;
  private Double price;
  private String pic;
  private String url;
  private java.util.Date created;
  private java.util.Date updated;


  public Long getId() {
    return id;
  }

  public void setId(Long id) {
    this.id = id;
  }


  public Long getSpu() {
    return spu;
  }

  public void setSpu(Long spu) {
    this.spu = spu;
  }


  public Long getSku() {
    return sku;
  }

  public void setSku(Long sku) {
    this.sku = sku;
  }


  public String getTitle() {
    return title;
  }

  public void setTitle(String title) {
    this.title = title;
  }


  public Double getPrice() {
    return price;
  }

  public void setPrice(Double price) {
    this.price = price;
  }


  public String getPic() {
    return pic;
  }

  public void setPic(String pic) {
    this.pic = pic;
  }


  public String getUrl() {
    return url;
  }

  public void setUrl(String url) {
    this.url = url;
  }


  public java.util.Date getCreated() {
    return created;
  }

  public void setCreated(java.util.Date created) {
    this.created = created;
  }


  public java.util.Date getUpdated() {
    return updated;
  }

  public void setUpdated(java.util.Date updated) {
    this.updated = updated;
  }

}

数据库表结构(2.2)

三: 配置application.properties进行连接数据库 (SpringDataJpa)

server.port=8081


#DB Configuration:
spring.datasource.driverClassName=com.mysql.jdbc.Driver
spring.datasource.url=jdbc:mysql://127.0.0.1:3306/crawler
spring.datasource.username=root
spring.datasource.password=12345

#JPA Configuration:
spring.jpa.database=MySQL
spring.jpa.show-sql=true

四: 编写Dao

五 ,我这里忽略了保存数据库service以及serviceimpl的操作

六,开始爬取数据,编写一个HttpUtils用来快速访问操作

package com.jd.crawler.utils;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.springframework.stereotype.Component;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.UUID;

/**
 * @author misterWei
 * @create 2018年10月06号:22点08分
 * @mailbox mynameisweiyan@gmail.com
 */
@Component
public class HttpUtils {
    //创建网络连接管理池
    private   PoolingHttpClientConnectionManager cm;

    public HttpUtils() {
        this.cm= new PoolingHttpClientConnectionManager();
        //最大连接数
        this.cm.setMaxTotal(200);
        //设置每个数据的连接数
        this.cm.setDefaultMaxPerRoute(20);
    }

    //需要定义两个Method方法
     //第一个用来访问连接

    public  String getHtml(String html){
        //创建访问对象
        CloseableHttpClient build = HttpClients.custom().setConnectionManager(cm).build();
       //声明httpget来进行请求响应
        HttpGet httpGet = new HttpGet(html);
        httpGet.setConfig(this.getrequestConfig());
        CloseableHttpResponse closeableHttpResponse = null;
        try {
            closeableHttpResponse = build.execute(httpGet);

            if (closeableHttpResponse.getStatusLine().getStatusCode()==200){
                   //此次状态请求200 成功 定义一个html的字符串页面
                String htmlStr = "";

                //并判断当前请求数据是否又返回数据内容等信息
                if (closeableHttpResponse.getEntity() != null){
                    htmlStr = EntityUtils.toString( closeableHttpResponse.getEntity(),"utf8");
                }
                return htmlStr;

            }


        }catch (Exception e){
            e.printStackTrace();
        }finally {
            try {
                if (                closeableHttpResponse != null){
                    closeableHttpResponse.close();

                }
            } catch (IOException e) {
                e.printStackTrace();
            }

        }
             return null;
    }
    //第二个用来保存图片

    public String getImage(String url){
        //创建访问对象
        CloseableHttpClient build = HttpClients.custom().setConnectionManager(cm).build();
        //声明httpget来进行请求响应
        HttpGet httpGet = new HttpGet(url);
        httpGet.setConfig(this.getrequestConfig());
        CloseableHttpResponse closeableHttpResponse = null;
        try {
            closeableHttpResponse = build.execute(httpGet);

            if (closeableHttpResponse.getStatusLine().getStatusCode()==200){
                //获取文件类型 MIME类型属性
             String endName =  url.substring(url.lastIndexOf("."));
             //根据UUID来生成文件名称
             String fileName = UUID.randomUUID().toString()+endName;
                OutputStream outputStream = new FileOutputStream("C:\\Users\\asus\\Desktop\\images\\"+fileName);
                //字节流写入指定的文件夹中
                closeableHttpResponse.getEntity().writeTo(outputStream);

                return fileName;


            }


        }catch (Exception e){
            e.printStackTrace();
        }finally {
            try {
               if ( closeableHttpResponse!=null){
                   closeableHttpResponse.close();
               }
            } catch (IOException e) {
                e.printStackTrace();
            }

        }
        return null;

    }





    //此方法用来设置连接参数

    private RequestConfig getrequestConfig(){
        RequestConfig requestConfig =  RequestConfig.custom().setConnectTimeout(3000)//设置创建连接的超时时间
        .setConnectionRequestTimeout(3000) //设置获取连接的超时时间
        .setSocketTimeout(1000).build();//设置连接的超时时间
    return requestConfig;

    }

}

七:主业务的编码流程:

 package com.jd.crawler.task;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.jd.crawler.service.CrawlerJdService;
import com.jd.crawler.pojo.JdItem;
import com.jd.crawler.utils.HttpUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.util.Date;
import java.util.List;

/**
 * @author misterWei
 * @create 2018年10月06号:22点39分
 * @mailbox mynameisweiyan@gmail.com
 */
@Component
public class JdItemTask {
    @Autowired
    private HttpUtils httpUtils;

    @Autowired  //这个就是一个保存数据库的service操作
    private CrawlerJdService crawlerJdService;
           //用来转化金额操作,
    public static final ObjectMapper objectMapper = new ObjectMapper();


    //主要逻辑的编写
     //间隔100秒执行一次
 @Scheduled(fixedDelay = 1000 * 100)
    public void process() throws Exception{

  String url =
          "https://search.jd.com/Search" +
                  "?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA&cid2=653&cid3=655&s=61&click=0" +
                  "&page=";//分页爬取数据

     for (int i = 0; i < 10; i=i+2) {
         String html = this.httpUtils.getHtml(url + i);
         //解析页面数据,并将解析的数据保存到数据库中
         this.parserHTML(html);
         System.out.println("保存成功!");
     }

 }

    private void parserHTML(String html) throws Exception {
     //使用Jsoup进行解析页面操作
        Document parse = Jsoup.parse(html);
        //根据页面的标签属性进行获取数据
        Elements select = parse.select("div#J_goodsList > ul > li");
        for (Element element : select) {
            //获取爬取数据的spu信息
            long spu = Long.parseLong(element.attr("data-spu"));
            Elements select1 = element.select("li.ps-item img");
            for (Element element1 : select1) {
                //获取sku数据
                long sku = Long.parseLong(element1.attr("data-sku"));
                //判断当前商品是否抓取
                JdItem jdItem = new JdItem();
                jdItem.setSku(sku);
                List<JdItem> jdItems = crawlerJdService.ifJditems(jdItem);
                if (jdItems.size()>0){
                    //如果大于0说明是有内容的也就是说是有数据的那就不执行操作,重新执行下一次循环
                    continue;
                }
                jdItem.setSpu(spu);

                //商品url
                jdItem.setUrl("https://item.jd.com/"+sku+".html");
                //创建时间
                jdItem.setCreated((java.util.Date) new Date());
                //更新时间
                jdItem.setUpdated((java.util.Date) new Date());
                //获取商品标题
                String html1 = this.httpUtils.getHtml(jdItem.getUrl());
                String title = Jsoup.parse(html1).select("div.sku-name").text();
                jdItem.setTitle(title);

                //获取商品价格
                String priceurl = "https://p.3.cn/prices/mgets?skuIds=J_"+sku;
                String html2 = this.httpUtils.getHtml(priceurl);
                double price = objectMapper.readTree(html2).get(0).get("p").asDouble();
                jdItem.setPrice(price);


                //获取图片价格信息
                String replace ="https:"+ element1.attr("data-lazy-img").replace("/n9/", "/n1/");
                String image = this.httpUtils.getImage(replace);
                System.out.println(image);
                jdItem.setPic(image);

                //保存数据到数据库中
                this.crawlerJdService.saveJdItem(jdItem);
            }

        }
    }
}

注意的是 本人在使用ObjectMapper 这个对象来自package com.fasterxml.jackson.databind

八,开启启动类启动获取数据:

我这里写的并不完善,但是在我的github中已经上传了,不仅仅有这些内容,还有webmagic小框架的搭建爬取数据,并且使用ElasticSearch作为搜索引擎,搭建,喜欢的朋友可以clone下来进行参考.
地址为:(https://github.com/selfconfidence/crawler-html.git)