1. 相关网址和库
网址
http://www.allitebooks.org/
需要用到的库
requests、beautifulsoup4
2. 代码实现
import requests
from lxml import etree
from bs4 import BeautifulSoup
import json
import csv
class BookSpider(object):
def __init__(self):
self.base_url = 'http://www.allitebooks.com/page/{}'
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36 Edg/83.0.478.45'
}
self.data_list = []
# 1.构建所有url
def get_url_list(self):
url_list = []
for i in range