xpath练习

‘’’
2020.5.28作业
xpath写法
‘’’
import requests
from lxml import etree
import socket
import time
import random
from pymongo import MongoClient

白嫖的代理ip

proxies = {
‘https’ : ‘58.218.201.74:2874’
}

设置代理

def getUserAgent():
“”"
:return: 随机返回一个浏览器请求头的User-Agent值
“”"
ua_list = [
“Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1”,
“Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11”,
“Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6”,
“Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6”,
‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36’
]
return random.choice(ua_list)

def get_page(url):
# 设置socket默认的等待时间，在read超时后能自动往下继续跑
socket.setdefaulttimeout(20)
haeders = {
“User-Agent”: getUserAgent()
}

response = requests.get(url, proxies=proxies, headers=haeders)
time.sleep(2)
# print(response.content)
# xpath写法
html = etree.HTML(response.content)
# 小说名
names = html.xpath('//div[@class="list_l_box"]//h4/a/@title')
# 图片
images = html.xpath('//div[@class="pic"]/a/img/@src')
# 主页地址
home_pages = html.xpath('//div[@class="list_l_box"]//h4/a/@href')
# 小说类别
novel_types = html.xpath('//div[@class="list_l_box"]//p/b/a[1]/text()')
# 作者
authors = html.xpath('//div[@class="list_l_box"]//p/b/a[2]/text()')
# 简介
explains = html.xpath('//div[@class="list_l_box"]//p[2]/text()')
# 发布时间
release_times = html.xpath('//div[@class="list_l_box"]//p[3]/font/text()')

response.close()

novel_list = []
for i in range(len(names)):
    novel = {
        '小说名':names[i],
        '图片':images[i],
        '首页地址':home_pages[i],
        '类别':novel_types[i],
        '作者':authors[i],
        '简介':explains[i],
        '发布时间':release_times[i]
    }
    novel_list.append(novel)
return novel_list

#mongo操作类
class mongo_demo():
def init(self):
self.client=MongoClient(‘mongodb://127.0.0.1:27017/’)
self.db=self.client[‘novels’]

def addMany(self, novel_list):     
    self.db.test.insert_many(novel_list)

if name == ‘main’:
mongo = mongo_demo()
for i in range(1,6):
if i > 1:
url = ‘http://www.txt80.com/dushi/index_’+ str(i) +’.html’
print(url)
novel_list = get_page(url)
print(novel_list)
mongo.addMany(novel_list)
else:
url = ‘http://www.txt80.com/dushi/index.html’
print(url)
novel_list = get_page(url)
print(novel_list)
mongo.addMany(novel_list)