总体设计
- 分析网页
- 确定需要用到的jar包，并通过pom.xml进行配置
- 创建一个电影实体类（Film），写入要爬取电影的相关属性，并用set和get方法封装
- 编写一个多线程爬取Top250的电影程序，并用一个数组（Film [ ]）存储电影信息
- 编写一个将数组（Film [ ]）写入Excel表格的函数
- 运行测试代码
详细设计
- 框架介绍
创建一个如下图的maven工程结构，其中com.douban.test.douban.until包下面存放的PageDownUntil类其功能是获取指定 URL 对应的 HTML内容，采用 Jsoup 解析。
entity包存放的是相关实体类，用来封装数据。例如，采集电影的影片名、导演、编剧、主演、评分等内容。
创建data文件夹主要是用来存放Top250的电影信息Excel表。
- 程序编写
2.1.jar包下载
基于 Maven 工程中的 pom.xml 下载该项目需要的相关 jar 包。
关键代码：
<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.13.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>4.1.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml -->
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>4.1.2</version>
</dependency>
2.2 entity
package entity;
public class Film {
private String title;// 电影名称
private String director;// 导演
private String scriptwriter;// 编剧
private String actors;// 主演
private String type;// 电影类型
private String country;// 制片国家
private String language;// 语言
private String time;// 上映时间
private String len;// 片长
private String other;// 又名
private String average; // 评分
private String rating_num;// 评分人数
private String theme;// 电影主题
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getDirector() {
return director;
}
public void setDirector(String director) {
this.director = director;
}
public String getScriptwriter() {
return scriptwriter;
}
public void setScriptwriter(String scriptwriter) {
this.scriptwriter = scriptwriter;
}
public String getActors() {
return actors;
}
public void setActors(String actors) {
this.actors = actors;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public String getCountry() {
return country;
}
public void setCountry(String country) {
this.country = country;
}
public String getLanguage() {
return language;
}
public void setLanguage(String language) {
this.language = language;
}
public String getTime() {
return time;
}
public void setTime(String time) {
this.time = time;
}
public String getLen() {
return len;
}
public void setLen(String len) {
this.len = len;
}
public String getOther() {
return other;
}
public void setOther(String other) {
this.other = other;
}
public String getAverage() {
return average;
}
public void setAverage(String average) {
this.average = average;
}
public String getRating_num() {
return rating_num;
}
public void setRating_num(String rating_num) {
this.rating_num = rating_num;
}
public String getTheme() {
return theme;
}
public void setTheme(String theme) {
this.theme = theme;
}
}
2.3 com.douban.test.douban.until包下的PageDownUntil类
这里主线程创建了一个固定长度为25的线程池，然后执行器调用getPage( )方法去执行10个任务获取页面的document。
getPage（）方法里面通过用设置变量start来拼接URL实现翻页操作；
在这个方法里面在用执行器去调用getFilms()方法执行爬取对应页面下的25部电影并存储到films数组里的任务。
Save（）方法是用来存储数据到Excel表；getWorkBook()方法是为了兼容Excel新旧版本。
最后成功运行代码，将爬取到的数据写进Excel表！
package com.douban.test.douban.until;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import entity.Film;
public class PageDownUntil {
static CountDownLatch cdl = new CountDownLatch(250);
public static void main(String[] args) throws IOException, InterruptedException {
Film films[] = new Film[250];
ExecutorService executor = Executors.newFixedThreadPool(25);
for (int i = 0; i < 250; i += 25) {
final int j = i;
executor.execute(() -> getPage(j, executor, films));
}
cdl.await();
executor.shutdown();
save(films);
}
public static void getPage(int start, ExecutorService executor, Film[] films) {
String url = "https://movie.douban.com/top250?start=" + start;
Document document;
try {
document = Jsoup.connect(url).header("Cookie",
"bid=7EGD1bBeKR0; ll=\"118283\"; __utma=30149280.1634505185.1623248959.1623248959.1623248959.1; __utmb=30149280.8.10.1623248959; __utmc=30149280; __utmz=30149280.1623248959.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmt=1; dbcl2=\"239058625:8jMCTPhp0mw\"; ck=5dlR; __utmv=30149280.23905; push_noty_num=0; push_doumail_num=0; ap_v=0,6.0; __gads=ID=ec2e6b335caa60df-22c6304a55c90008:T=1623249142:S=ALNI_MbI-YaCg0inouoLg3p-VYwhNhwB6A; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1623249174%2C%22https%3A%2F%2Fwww.douban.com%2F%22%5D; _pk_id.100001.4cf6=6df12779acf17f74.1623249174.1.1623249197.1623249174.; _pk_ses.100001.4cf6=*; __utma=223695111.936811413.1623249176.1623249176.1623249176.1; __utmb=223695111.0.10.1623249176; __utmc=223695111; __utmz=223695111.1623249176.1.1.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; _vwo_uuid_v2=D10A1BE593D38700888532337DC26DB0B|eba5b6b34248470f6c125e2a9376a9b6")
.header("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0")
.timeout(Integer.MAX_VALUE).get();
for (int i = 0; i < 25; i++) {
final int j = i;
executor.execute(() -> getFilms(document, start, j, films));
}
} catch (IOException e) {
// TODO 自动生成的 catch 块
e.printStackTrace();
}
}
public static void getFilms(Document document, int start, int index, Film[] films) {
// TODO 自动生成的方法存根
Element element = document.select("div.info").get(index);
String tempurl = element.select(".hd>a").attr("href");
String average = element.select(".star .rating_num").text();
String rating_num = element.select(".star span:last-child").text();
rating_num = rating_num.substring(0, rating_num.length() - 3);
String theme = element.select(".inq").text();
if (!theme.isEmpty()) {
theme = theme.substring(0, theme.length() - 1);
}
Document tempdocument = null;
try {
tempdocument = Jsoup.connect(tempurl).header("Cookie",
"bid=7EGD1bBeKR0; ll=\"118283\"; __utma=30149280.1634505185.1623248959.1623248959.1623248959.1; __utmb=30149280.8.10.1623248959; __utmc=30149280; __utmz=30149280.1623248959.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utmt=1; dbcl2=\"239058625:8jMCTPhp0mw\"; ck=5dlR; __utmv=30149280.23905; push_noty_num=0; push_doumail_num=0; ap_v=0,6.0; __gads=ID=ec2e6b335caa60df-22c6304a55c90008:T=1623249142:S=ALNI_MbI-YaCg0inouoLg3p-VYwhNhwB6A; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1623249174%2C%22https%3A%2F%2Fwww.douban.com%2F%22%5D; _pk_id.100001.4cf6=6df12779acf17f74.1623249174.1.1623249197.1623249174.; _pk_ses.100001.4cf6=*; __utma=223695111.936811413.1623249176.1623249176.1623249176.1; __utmb=223695111.0.10.1623249176; __utmc=223695111; __utmz=223695111.1623249176.1.1.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/; _vwo_uuid_v2=D10A1BE593D38700888532337DC26DB0B|eba5b6b34248470f6c125e2a9376a9b6")
.header("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0")
.timeout(Integer.MAX_VALUE).get();
} catch (IOException e) {
// TODO 自动生成的 catch 块
e.printStackTrace();
}
String director = tempdocument.select("div#info span:first-child span.attrs>a").text();
String scriptwriter = tempdocument.select("div#info span:nth-of-type(2) span.attrs").text();
String actors = tempdocument.select("div#info span.actor span.attrs").text();
String type = String.join("/", tempdocument.select("div#info span[property='v:genre']").eachText());
Elements elements = tempdocument.select("div#info>span.pl");
String country = elements.get(1).nextSibling().toString().trim();
String language = elements.get(2).nextSibling().toString().trim();
String other = elements.get(5).nextSibling().toString().trim();
String title = tempdocument.select("div#content h1").text();
String time = String.join("/",
tempdocument.select("div#info span[property='v:initialReleaseDate']").eachText());
String len = tempdocument.select("div#info span[property='v:runtime']").text();
Film film = new Film();
film.setTitle(title);
film.setOther(other);
film.setDirector(director);
film.setScriptwriter(scriptwriter);
film.setActors(actors);
film.setType(type);
film.setCountry(country);
film.setLanguage(language);
film.setTime(time);
film.setLen(len);
film.setAverage(average);
film.setRating_num(rating_num);
film.setTheme(theme);
films[index + start] = film;
System.out.println((start + index) + "\t" + film.getTitle());
cdl.countDown();
}
public static void save(Film[] films) throws IOException {
String pathname = "data/top250.xlsx";
File file = new File(pathname);
OutputStream outputStream = new FileOutputStream(file);
Workbook workbook = getWorkBook(file);
Sheet sheet = workbook.createSheet("Sheet1");
String header[] = "top250 影片名称 又名 导演 编剧 主演 影片类型 制片国家/地区 语言 上映时间 片长 评分 评论人数 主题".split(" ");
Row row = sheet.createRow(0);
for (int i = 0; i < 14; i++) {
row.createCell(i).setCellValue(header[i]);
}
for (int i = 1; i <= 250; i++) {
row = sheet.createRow(i);
row.createCell(0).setCellValue(i);
row.createCell(1).setCellValue(films[i - 1].getTitle());
row.createCell(2).setCellValue(films[i - 1].getOther());
row.createCell(3).setCellValue(films[i - 1].getDirector());
row.createCell(4).setCellValue(films[i - 1].getScriptwriter());
row.createCell(5).setCellValue(films[i - 1].getActors());
row.createCell(6).setCellValue(films[i - 1].getType());
row.createCell(7).setCellValue(films[i - 1].getCountry());
row.createCell(8).setCellValue(films[i - 1].getLanguage());
row.createCell(9).setCellValue(films[i - 1].getTime());
row.createCell(10).setCellValue(films[i - 1].getLen());
row.createCell(11).setCellValue(films[i - 1].getAverage());
row.createCell(12).setCellValue(films[i - 1].getRating_num());
row.createCell(13).setCellValue(films[i - 1].getTheme());
}
workbook.write(outputStream);
workbook.close();
outputStream.close();
System.out.println("finish!");
}
private static Workbook getWorkBook(File file) {
// TODO 自动生成的方法存根
Workbook workbook = null;
if (file.getName().endsWith(".xls")) {
workbook = new HSSFWorkbook();
} else if (file.getName().endsWith(".xlsx")) {
workbook = new XSSFWorkbook();
}
return workbook;
}
}
2413
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
