采用requests库爬取中国天气网七日天气信息,爬取完毕后将信息写入文件
import requests
from lxml import etree
def getWeatherInfo(url):
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'cookie': '登录网站后cookie填这里'
}
response = requests.get(url, headers=headers)
response.encoding = response.apparent_encoding
html = etree.HTML(response.text)
all_weather = html.xpath('/html/body/div[4]/div[2]/div/div/div[2]/div')
dates=html.xpath('/html/body/div[4]/div[2]/div/div/ul[2]/li')
date=[]
for d in dates:
date.append(d.xpath('./text()')[0].split('(')[1].strip(')'))
for index1,info in enumerate(all_weather):
conMidtab2 = info.xpath("./div")
for conmid2 in conMidtab2:
tr_list = conmid2.xpath('./table/tr')[2:]
for index, tr in enumerate(tr_list):
if index == 0:
provice = tr.xpath('./td[1]/a/text()')[0]
city = tr.xpath('./td[2]/a/text()')[0]
weather = tr.xpath('./td[3]/text()')[0]
wind = tr.xpath('./td[4]/span[1]/text()')[0]
wind_grade = tr.xpath('./td[4]/span[2]/text()')[0]
temperature = tr.xpath('./td[5]/text()')[0]
print(provice, city, weather, wind, wind_grade, temperature,date[index1])
else:
city = tr.xpath('./td[1]/a/text()')[0]
weather = tr.xpath('./td[2]/text()')[0]
wind = tr.xpath('./td[3]/span[1]/text()')[0]
wind_grade = tr.xpath('./td[3]/span[2]/text()')[0]
temperature = tr.xpath('./td[4]/text()')[0]
print(provice, city, weather, wind, wind_grade, temperature,date[index1])
with open("dayofzgweather.csv", 'a', encoding='utf-8') as f:
f.write("{},{},{},{},{},{},{}".format(provice, city, weather, wind, wind_grade, temperature,date[index1]))
f.write('\n')
if __name__ == '__main__':
dic={'华北':'hb','东北':'db','华东':'hd','华中':'hz','华南':'hn','西北':'xb','西南':'xn'}
area=input("请输入地区:")
url=f'http://www.weather.com.cn/textFC/{dic[area]}.shtml'
print(url)
getWeatherInfo(url)