使用Python爬取天气网2013-2019天气数据

使用Python爬取天气网天气数据

使用Python的requests 和BeautifulSoup模块，Python 3.7可在命令行中直接使用pip进行模块安装。

  # pip install requests 
  # pip install bs4

爬取云南省昆明市2013-2019年气象数据代码如下：

#encoding:utf-8                                                                  
import requests                                                                      
from bs4 import BeautifulSoup
n = ['13','14','15','16','17','18','19']
y = ['01','02','03','04','05','06','07','08','09','10','11','12']
urls =[]
for n1 in n:
    for y1 in y:
        
        ur='http://lishi.tianqi.com/kunming/20'+n1+y1+'.html'
        urls.append(ur)
#print(urls)  
file = open('kunming_weather.csv','w')
#这里的header是我自己的，已经被我注释，需要改成自己的header，一定要改，要不然爬不了
header = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/*******(KHTML, like Gecko) Chrome/***************',
    'Cookie':'cityPy=*******; cityPy_expire=********; UM_distinctid=***********; CNZZDATA1275796416=18*******************4; Hm_lvt_ab6a683aa97a52202eab5b3a9042a8d2=*********,15****08,15857***4,1585***2; Hm_lpvt_ab6a683aa97a52202eab5***********6834'
    }
for url in urls:                                                                     
    response = requests.get(url,headers = header)                                                     
    soup = BeautifulSoup(response.text, 'html.parser')                               
    weather_list = soup.select('div[class="tian_three"]')
    #print(weather_list)  
     
    for weather in weather_list:                                                     
        weather_date = weather.select('ul[class="thrui"]')
        
        ul_list = weather.select('li')                                                                                                                    
        for ul in ul_list:                                                           
            li_list= ul.select('div')                                                 
            str=""                                                                   
            for li in li_list:
                print(li.string)
                if(li.string==None):
                    str =str+'无数据'+','
                else:
                    str =str+li.string+','
                
                
            if (str!=' '):                                                                 
                file.write(str+'\n')                                              
                                                                 
file.close() 

结果：