用xpath提取三国杀武将信息

用xpath爬取百度百科上三国杀词条下的武将信息

import requests
from lxml import etree

url = 'https://baike.baidu.com/item/%E4%B8%89%E5%9B%BD%E6%9D%80%E5%8D%A1%E7%89%8C%E8%B5%84%E6%96%99/4348115?fr=aladdin#2'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3872.400 QQBrowser/10.8.4455.400'
}
resp = requests.get(url=url,headers=headers)
# print(resp.text)
html = etree.HTML(resp.text)
table_list = html.xpath("/html/body/div[3]/div[2]/div/div[1]/table")[1:5]
fp = open("./三国杀武将.txt",'w',encoding='utf-8')
for table in table_list:
    trs = table.xpath("./tr")[1:]
    # print(trs)
    for tr in trs:
        name = tr.xpath("./td[2]/div/text()")[0]
        jineng = "\n".join(tr.xpath("./td[3]/div/text()"))
        print(name)
        print((jineng))
        fp.write(name+'\n'+jineng+'\n')
        # fp.write(jineng)
print("over!!!")
fp.close()