java 爬虫数据整合_java实现maven项目最简单的网络爬虫并导出爬取的数据

一、pom包引入

org.apache.httpcomponents

httpclient

4.5.3

org.apache.httpcomponents

httpmime

4.5.3

org.jsoup

jsoup

1.10.2

org.apache.poi

poi

3.16

org.apache.poi

poi-ooxml-schemas

3.16

org.apache.poi

poi-excelant

3.16

org.apache.poi

poi-examples

3.16

二、创建实体类Product  属性可以根据自已的业务定义 (生成set get 方法   快捷键 alt+insert )

private String productName; //品名

private String specs;//规格

private String brand; //品牌

private String lowerPrice; //价格

private String OfferArea; //报价区域

private String OfferCompany; //报价企业

private String offerDate; //报价日

三、抓取数据业务代码

import java.io.FileNotFoundException;

import java.io.IOException;

import java.util.ArrayList;

import java.util.List;

import org.jsoup.Jsoup;

import org.jsoup.nodes.Document;

import org.jsoup.nodes.Element;

import org.jsoup.select.Elements;

import com.dark.pojo.Product;

import com.dark.util.POItoExcel;

public class Reptile {

public static void main(String[] args) throws FileNotFoundException, IOException {

List list=getInfor("https://xfm.dazpin.com/prices/"); //要抓取数据的 url 返回多条数据，

POItoExcel.toExcel(list); //导出到Excel 根据业务可以存入数据库

}

public static List getInfor(String url){

List proList=new ArrayList();

try {

Document doc=Jsoup.connect(url).get(); //使用Jsoup 解析HTML

Elements pages=doc.select(".pages a"); //.pages a HTML类选择器 pages 下面的 a 标签，这里是获取最大页数

for (int i = 0; i < pages.size(); i++) { //遍历页数 抓取每页数据

url = "https://xfm.dazpin.com/prices/"+ (i + 1) +".html";

doc=Jsoup.connect(url).get();

Elements table=doc.select(".b-j-con table:first-child"); // .b-j-con 找到要抓取数据的table表 (.xx 代表类选择器)

Elements tbody=table.select("tbody");//获取到表单的体

Elements trList=tbody.select("tr");//找到 tr 标签 里面的数据

trList.remove(0);

for(Element tr:trList){ //遍历抓取数据

Elements tdList=tr.select("td");

Product product=new Product();

//该td 标签下面有二级标签

product.setProductName(tdList.get(0).select("p a").html().toString());//品名

product.setSpecs(tdList.get(1).html().toString());//规格

product.setBrand(tdList.get(2).html().toString());//品牌

product.setLowerPrice(tdList.get(3).html().toString());//价格

product.setOfferArea(tdList.get(4).select("p").html().toString());//报价区域

product.setOfferCompany(tdList.get(5).select("p").html().toString());//报价企业

product.setOfferDate(tdList.get(6).html().toString());

proList.add(product); //数据封装List

}

}

} catch (IOException e) {

e.printStackTrace();

}

return proList;

}

}

四、导出数据 (根据业务可以存入数据库)

import java.io.File;

import java.io.FileNotFoundException;

import java.io.FileOutputStream;

import java.io.IOException;

import java.util.List;

import org.apache.poi.ss.usermodel.Cell;

import org.apache.poi.ss.usermodel.CellStyle;

import org.apache.poi.ss.usermodel.HorizontalAlignment;

import org.apache.poi.ss.usermodel.Row;

import org.apache.poi.ss.usermodel.VerticalAlignment;

import org.apache.poi.ss.util.CellRangeAddress;

import org.apache.poi.xssf.usermodel.XSSFCell;

import org.apache.poi.xssf.usermodel.XSSFRow;

import org.apache.poi.xssf.usermodel.XSSFSheet;

import org.apache.poi.xssf.usermodel.XSSFWorkbook;

import com.dark.pojo.Product;

public class POItoExcel {

public static void toExcel(List list) throws FileNotFoundException, IOException{

XSSFWorkbook workBook=new XSSFWorkbook();

XSSFSheet sheet=workBook.createSheet();

CellRangeAddress cra=new CellRangeAddress(0,1, 0, 6);

sheet.addMergedRegion(cra);

Row row2=sheet.createRow(0);

Cell cell=row2.createCell(0);

cell.setCellValue("新凤鸣报价单");

CellStyle cs=workBook.createCellStyle();

cs.setAlignment(HorizontalAlignment.CENTER);

cs.setVerticalAlignment(VerticalAlignment.CENTER);

cs.setFillBackgroundColor((short) 59);

cell.setCellStyle(cs);

Row row=sheet.createRow(2);

Cell cell11=row.createCell(0);

cell11.setCellValue("品名");

Cell cell22=row.createCell(1);

cell22.setCellValue("规格型号");

Cell cell33=row.createCell(2);

cell33.setCellValue("品牌");

Cell cell44=row.createCell(3);

cell44.setCellValue("价格");

Cell cell55=row.createCell(4);

cell55.setCellValue("报价区域");

Cell cell66=row.createCell(5);

cell66.setCellValue("报价企业");

Cell cell77=row.createCell(6);

cell77.setCellValue("报价时间");

for(int i=0;i

XSSFRow row4=sheet.createRow(i+3);

XSSFCell cell1=row4.createCell(0);

XSSFCell cell2=row4.createCell(1);

XSSFCell cell3=row4.createCell(2);

XSSFCell cell4=row4.createCell(3);

XSSFCell cell5=row4.createCell(4);

XSSFCell cell6=row4.createCell(5);

XSSFCell cell7=row4.createCell(6);

cell1.setCellValue(list.get(i).getProductName());

cell2.setCellValue(list.get(i).getSpecs());

cell3.setCellValue(list.get(i).getBrand());

cell4.setCellValue(list.get(i).getLowerPrice());

cell5.setCellValue(list.get(i).getOfferArea());

cell6.setCellValue(list.get(i).getOfferCompany());

cell7.setCellValue(list.get(i).getOfferDate());

}

workBook.write(new FileOutputStream(new File("E:\\\\测试.xls")) );

workBook.close();

}

}