使用xpath爬取我的博客的发帖信息并保存在csv文件中
#! /usr/bin/env python
# -*- coding:utf8 -*-
import requests
import csv
from lxml import etree
url = 'https://blog.csdn.net/qiaoenshi'
headers = {"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"}
data = requests.get(url, headers=headers)
data.encoding='utf-8'
html = etree.HTML(data.text)
hrefs = html.xpath('//div[@class="article-list"]/div/h4/a/@href')
titles = html.xpath('//div[@class="article-list"]/div/h4/a/text()')
print(titles)
# 去除换行符
result = []
for i in range(len(titles)):
if (i % 2) == 1:
print(titles[i])
result.append(titles[i])
print(result)
xinxi_list = []
for i in range(len(hrefs)):
xinxi = {"链接": hrefs[i], "标题": result[i]}
xinxi_list.append(xinxi)
print(xinxi_list)
with open('博客发帖信息.csv', 'w', encoding='utf-8') as f:
w = csv.DictWriter(f, fieldnames=["链接", "标题"])
w.writeheader()
w.writerows(xinxi_list) # 这里一定不能用writerow会报错的,血泪教训
这里面遇到的问题是标题有换行符,本来想用正则解决,但是正则不太熟悉,就用一个循环把空行去掉保存到一个新的list中
报文件的截图