spider爬虫demo

通过 XPath，提取小节名称与以及小节名称对应的链接地址。然后将其保存为 excel 文件。

目标网址：https://www.runoob.com/html/html-tutorial.html

import requests
import lxml.etree as le
import pandas as pd

url = 'https://www.runoob.com/html/html-tutorial.html'
x = '//div[@id = "leftcolumn"]/a/text()'
y = '//div[@id="leftcolumn"]/a/@href'
content = requests.get(url = url).content
contentx = le.HTML(content)

titles = contentx.xpath(x)
urls = contentx.xpath(y)

ti = []
ul = []

for i in titles:
    ti.append(i.strip())
for l in urls:
    ul.append(l.strip())
data = list(map(lambda key, value: [key, value],ti,ul))
data = pd.DataFrame(data)
# print(data)
data.columns = ['title','url']
write = pd.ExcelWriter('菜鸟_title_url.xlsx')
data.to_excel(write,sheet_name='title_url')
write.save()