‘’’
2020.5.28作业
xpath写法
‘’’
import requests
from lxml import etree
import socket
import time
import random
from pymongo import MongoClient
白嫖的代理ip
proxies = {
‘https’ : ‘58.218.201.74:2874’
}
设置代理
def getUserAgent():
“”"
:return: 随机返回一个浏览器请求头的User-Agent值
“”"
ua_list = [
“Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1”,
“Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11”,
“Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6”,
“Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6”,
‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36’
]
return random.choice(ua_list)
def get_page(url):
# 设置socket默认的等待时间,在read超时后能自动往下继续跑
socket.setdefaulttimeout(20)
haeders = {
“User-Agent”: getUserAgent()
}
response = requests.get(url, proxies=proxies, headers=haeders)
time.sleep(2)
# print(response.content)
# xpath写法
html = etree.HTML(response.content)
# 小说名
names = html.xpath('//div[@class="list_l_box"]//h4/a/@title')
# 图片
images = html.xpath('//div[@class="pic"]/a/img/@src')
# 主页地址
home_pages = html.xpath('//div[@class="list_l_box"]//h4/a/@href')
# 小说类别
novel_types = html.xpath('//div[@class="list_l_box"]//p/b/a[1]/text()')
# 作者
authors = html.xpath('//div[@class="list_l_box"]//p/b/a[2]/text()')
# 简介
explains = html.xpath('//div[@class="list_l_box"]//p[2]/text()')
# 发布时间
release_times = html.xpath('//div[@class="list_l_box"]//p[3]/font/text()')
response.close()
novel_list = []
for i in range(len(names)):
novel = {
'小说名':names[i],
'图片':images[i],
'首页地址':home_pages[i],
'类别':novel_types[i],
'作者':authors[i],
'简介':explains[i],
'发布时间':release_times[i]
}
novel_list.append(novel)
return novel_list
#mongo操作类
class mongo_demo():
def init(self):
self.client=MongoClient(‘mongodb://127.0.0.1:27017/’)
self.db=self.client[‘novels’]
def addMany(self, novel_list):
self.db.test.insert_many(novel_list)
if name == ‘main’:
mongo = mongo_demo()
for i in range(1,6):
if i > 1:
url = ‘http://www.txt80.com/dushi/index_’+ str(i) +’.html’
print(url)
novel_list = get_page(url)
print(novel_list)
mongo.addMany(novel_list)
else:
url = ‘http://www.txt80.com/dushi/index.html’
print(url)
novel_list = get_page(url)
print(novel_list)
mongo.addMany(novel_list)