爬虫task2

爬虫2
XPath爬取丁香论坛回复评论

import requests
from lxml import etree
class Dingxiangyuan(object):
def init(self):
self.base_url = ‘http://www.dxy.cn/bbs/thread/626626’
self.headers = {“User-Agent”: “Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko”}
# 数据集合 list
self.data_list = []
# 1.发请求
def send_request(self, tencent_params):
try:
data = requests.get(self.base_url, headers=self.headers, params=tencent_params).content.decode()
return data
except Exception as e:
print(e)
# 2.解析数据 xpath
def analysis_data(self, data):
# 1.转换成 解析类型
html_data = etree.HTML(data)
# 2. 解析数据 获取所有 tr 的标签对象 list
dict_data = html_data.xpath(’//tbody//td[@class=“postbody”]’)
self.data_list.append(dict_data)
# 3.保存数据
def write_file(self):
with open(‘dxy.txt’, ‘w’) as f:
for content in self.data_list:
for a in content:
f.write(a)
# 4.调度 run
def run(self):
# 1.拼参数
dxy_prams = {
“keywords”: “python”,
“lid”: “2156”,
“tid”: “0”,
“start”: “0”
}
# 2.发送请求
data = self.send_request(dxy_params=dxy_prams)
# 3.解析数据
self.analysis_data(data)
# 4.保存
self.write_file()
if name == ‘main’:
Dingxiangyuan().run()