python爬取所有的城市名称

爬取所有的城市名称
import requests
from lxml import etree

if name == ‘main’:
url=‘https://www.aqistudy.cn/historydata/’
header = {
‘user_Agent’: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36’
}
page_text=requests.get(url=url,headers=header).text
#数据解析
tree=etree.HTML(page_text)

# title_list=tree.xpath("//ul[@class='unstyled']/div[1]")
#print(title_list)

fp=open('./cityname.txt','w',encoding='utf-8')
# for i in range(1,23):
#     li_list=tree.xpath("//ul[@class='unstyled'][i]/div[2]/li")
#     for title in title_list:
#         city_title=title.xpath("./b/text()")[0]
#         city_title=str(city_title)+'\n'
#         fp.write(city_title)
#         for i in range(1, 23):
#             # li_list=tree.xpath("//div[@class='bottom']/ul[i]/div[2]/li")
#             li_list = tree.xpath("//div[@class='bottom']/ul")[i]
#             # print(li_list)
#             for li in li_list:
#                 city_name=li.xpath("./div[2]/li/a/text()")[0]
#                 #print(city_name)
#                 city_name=str(city_name)+'\t'
#                 fp.write(city_name)
#                 print(city_name,'爬取成功')
#     fp.write('\n')
a_list=tree.xpath("//div[@class='bottom']/ul/li/a | //div[@class='bottom']/ul/div[2]/li/a")
for a in a_list:
    city=a.xpath('./text()')[0]
    city=str(city)+'\t'
    fp.write(city)
    print(city,"爬取成功")

因为在网页中热门城市和所有城市不是在一个class标签中，而且标签的层级也不相同，所以可以用两种方法
方法一就是一个一个爬取，就需要分别解析两个不同层级的标签，即是代码中被注释的部分
方法二就是把两个不同层级的标签写在一起，就是代码中未被注释的部分
tree.xpath("//div[@class=‘bottom’]/ul/li/a | //div[@class=‘bottom’]/ul/div[2]/li/a")