用xpath爬取百度百科上三国杀词条下的武将信息
import requests
from lxml import etree
url = 'https://baike.baidu.com/item/%E4%B8%89%E5%9B%BD%E6%9D%80%E5%8D%A1%E7%89%8C%E8%B5%84%E6%96%99/4348115?fr=aladdin#2'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3872.400 QQBrowser/10.8.4455.400'
}
resp = requests.get(url=url,headers=headers)
# print(resp.text)
html = etree.HTML(resp.text)
table_list = html.xpath("/html/body/div[3]/div[2]/div/div[1]/table")[1:5]
fp = open("./三国杀武将.txt",'w',encoding='utf-8')
for table in table_list:
trs = table.xpath("./tr")[1:]
# print(trs)
for tr in trs:
name = tr.xpath("./td[2]/div/text()")[0]
jineng = "\n".join(tr.xpath("./td[3]/div/text()"))
print(name)
print((jineng))
fp.write(name+'\n'+jineng+'\n')
# fp.write(jineng)
print("over!!!")
fp.close()