使用Python爬取天气网天气数据
使用Python的requests 和BeautifulSoup模块,Python 3.7可在命令行中直接使用pip进行模块安装。
# pip install requests
# pip install bs4
爬取云南省昆明市2013-2019年气象数据代码如下:
#encoding:utf-8
import requests
from bs4 import BeautifulSoup
n = ['13','14','15','16','17','18','19']
y = ['01','02','03','04','05','06','07','08','09','10','11','12']
urls =[]
for n1 in n:
for y1 in y:
ur='http://lishi.tianqi.com/kunming/20'+n1+y1+'.html'
urls.append(ur)
#print(urls)
file = open('kunming_weather.csv','w')
#这里的header是我自己的,已经被我注释,需要改成自己的header,一定要改,要不然爬不了
header = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/*******(KHTML, like Gecko) Chrome/***************',
'Cookie':'cityPy=*******; cityPy_expire=********; UM_distinctid=***********; CNZZDATA1275796416=18*******************4; Hm_lvt_ab6a683aa97a52202eab5b3a9042a8d2=*********,15****08,15857***4,1585***2; Hm_lpvt_ab6a683aa97a52202eab5***********6834'
}
for url in urls:
response = requests.get(url,headers = header)
soup = BeautifulSoup(response.text, 'html.parser')
weather_list = soup.select('div[class="tian_three"]')
#print(weather_list)
for weather in weather_list:
weather_date = weather.select('ul[class="thrui"]')
ul_list = weather.select('li')
for ul in ul_list:
li_list= ul.select('div')
str=""
for li in li_list:
print(li.string)
if(li.string==None):
str =str+'无数据'+','
else:
str =str+li.string+','
if (str!=' '):
file.write(str+'\n')
file.close()
结果: