京东商城网页数据爬取

对于做用户研究的同事经常需要去各个平台搜索“竞品”、“近品”等的信息，特别是用户购买后的评论信息，然后做研究分析，笔者的一个朋友曾为了搜集这些数据，在网页上一条条的去搜集，花费大量的时间。于是笔者写了这个demo供朋友使用。

首先，需要引入几个jar包

    gson.jar 和 poi.jar包

<!-- 解析返回的评论信息--> 
<dependency>
	<groupId>com.google.code.gson</groupId>
	<artifactId>gson</artifactId>
    <version>2.2.4</version>
</dependency>

<!-- 把信息写入到excel中使用--> 
<dependency>
   <groupId>org.apache.poi</groupId>
   <artifactId>poi</artifactId>
   <version>4.0.1</version>
</dependency>

下面我们给出实现代码

首先定义接口：  excel输出的简单封装

package com.zybank.spring.gson.framework.service;

import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;

import com.zybank.spring.gson.framework.bean.ExcelBean;

/**
* @author：zhangfd
* @version 1.0.0
* @date  2018年12月5日 上午9:42:40
* @description
*/
public interface ExcelInterface {

	public HSSFSheet createSheet(ExcelBean excelBean);
	
	public void writeExcel(String destUrl,HSSFWorkbook workbook);
}

package com.zybank.spring.gson.framework.bean;

import java.io.Serializable;
import java.util.List;

import org.apache.poi.hssf.usermodel.HSSFWorkbook;

/**
* @author：zhangfd
* @version 1.0.0
* @date  2018年12月5日 上午9:43:30
* @description
*/
public class ExcelBean implements Serializable{

	private static final long serialVersionUID = 1L;

	private String sheetName;//所定义的shell的名称
		
	private List<String> columnName;//列名
	
	private HSSFWorkbook workbook; //该sheet所属的workbook

	public String getSheetName() {
		return sheetName;
	}

	public void setSheetName(String sheetName) {
		this.sheetName = sheetName;
	}
	
	public List<String> getColumnName() {
		return columnName;
	}

	public void setColumnName(List<String> columnName) {
		this.columnName = columnName;
	}

	public HSSFWorkbook getWorkbook() {
		return workbook;
	}

	public void setWorkbook(HSSFWorkbook workbook) {
		this.workbook = workbook;
	}

	
	
}

接口的实现类：

package com.zybank.spring.gson.framework.service.imp;

import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;

import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;

import com.zybank.spring.gson.framework.bean.ExcelBean;
import com.zybank.spring.gson.framework.service.ExcelInterface;
/**
 * 
 * @author zhangfd
 * excel文件的简单整合
 */
public  class ExcelService implements ExcelInterface{

	/**
	 * 创建excel文件里的一个sheet，
	 * 并且根据传入的参数，把第一行赋值标题，设置每列的长度大小
	 * @return 返回这个sheet，后面需要对具体的内容赋值
	 */
	public HSSFSheet createSheet(ExcelBean excelBean) {
		if(null == excelBean) throw new RuntimeException("请求参数不能为空");
		
		if(null == excelBean.getWorkbook());
		HSSFWorkbook workbook = excelBean.getWorkbook();
		HSSFSheet sheet = workbook.createSheet(excelBean.getSheetName());
		HSSFRow row = sheet.createRow(0);

		List<String> columnNameList = excelBean.getColumnName();
		for(int i=0;i<columnNameList.size();i++){
			 sheet.setColumnWidth(i, 20 * 256);
			 HSSFCell cell = row.createCell(i);
			 cell.setCellValue(columnNameList.get(i));
		}
		
		return sheet;
	}
	
	/**
	 * @param destUrl 输出excel文件的路径，包括文件名
	 * @param workbook 所有创建的sheet对应的workbook
	 */
	@Override
	public void writeExcel(String destUrl, HSSFWorkbook workbook) {
	    //将文件保存到指定的位置
	    try {
	        FileOutputStream fos = new FileOutputStream(destUrl);
	        workbook.write(fos);
	        System.out.println("写入成功");
	        fos.close();
	    } catch (IOException e) {
	        e.printStackTrace();
	    }
		
	}
	
	
	
	
	public static void main(String[] args) {
		
        HSSFWorkbook workbook = new HSSFWorkbook();
		
		ExcelInterface excel = new ExcelService();
		ExcelBean excelBean = new ExcelBean();
		excelBean.setWorkbook(workbook);
		excelBean.setSheetName("sheet1");
		String[] s = {"colum1","colum2","colum1","colum3"};
		excelBean.setColumnName(Arrays.asList(s));
		HSSFSheet sheet = excel.createSheet(excelBean);
		//仅仅写一行用与测试
		HSSFRow row1 = sheet.createRow(1);
		row1.createCell(0).setCellValue("zhansan");
		row1.createCell(1).setCellValue("zhansan1");
		row1.createCell(2).setCellValue("zhansan2");
		row1.createCell(3).setCellValue("zhansan3");
		
		excelBean.setSheetName("sheet2");
		String[] s1 = {"colum11","colum12","colum11","colum13"};
		excelBean.setColumnName(Arrays.asList(s1));
		sheet = excel.createSheet(excelBean);
		//仅仅写一行用与测试
		row1 = sheet.createRow(1);
		row1.createCell(0).setCellValue("lisi");
		row1.createCell(1).setCellValue("lisi1");
		row1.createCell(2).setCellValue("lisi2");
		row1.createCell(3).setCellValue("lisi3");
		
		excel.writeExcel("C:\\Users\\lenovo\\Desktop\\fileName.xls", workbook);
		
		
	}
}

定义单个产品页面的解析方法接口

package com.zybank.spring.gson.framework.service;

import java.io.IOException;
import java.util.List;

import org.apache.http.HttpException;

import com.zybank.spring.gson.framework.bean.DJCommentSummary;
import com.zybank.spring.gson.framework.bean.DJCommentsBean;

/**
* @author：zhangfd
* @version 1.0.0
* @date  2018年12月6日 上午11:21:53
* @description
*/
public interface AnalySiteInfoInterface {

	public List<DJCommentsBean> analySiteCommentsInfo(String path) throws HttpException, IOException;
	
	public DJCommentSummary analySiteProductCommentSummaryInfo(String path) throws HttpException, IOException;

}

package com.zybank.spring.gson.framework.bean;

import java.io.Serializable;

/**
* @author：zhangfd
* @version 1.0.0
* @date  2018年12月6日 上午11:22:28
* @description
*/
public class DJCommentsBean implements Serializable{

	private static final long serialVersionUID = 1L;

	private String  userId;//用户id
	
	private String  creationTime;//评论时间
	
	private String content;//评论内容

	private String nickname;//会员号
	
	private String userLevelName;//会员等级
	
	private String score;//星星数
	
	private String productName;//产品名称
	
	private String color;//产品颜色
	
	private String afterUserCommentTime;//追加评论时间
	
	private String hAfterUserComme