爬取中国天气网所有城市的数据,并进行可视化
注意:
1、中国天气网没有设置反爬虫机制
2、对于香港的数据,再网页源码中有<table>,无</table>,再浏览器的元素中,是浏览器自动加的
此时用‘lxml’解析器会出错(因为lxml容错能力较差),此时应该用‘html5lib’解析器(容错能力较强)
安装方法:pip instail html5lib
语句:soup = BeautifulSoup(text,'html5lib')
# 需求:爬取中国天气网各城市最低气温
import requests
from bs4 import BeautifulSoup
from pyecharts import Bar # 制作网页显示的图表
ALL_DATA = []
def parse_page(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36'
}
response = requests.get(url, headers = headers)
text = response.content.decode('utf-8')
# html5lib: 容错能力强,能解决“香港”的代码中无</table>问题
# pip install html5lib
soup = BeautifulSoup(text, 'html5lib')
conMidtab = soup.fid('div', class='conMidtab')
tables = conMidtab.find_all('table')
for table in tables:
trs = table.find_all('tr')[2:]
for index, tr in enumerate(trs):
tds = tr.find_all('td')
city_td = tds[0] # 城市名称
if index = 0:
city_td = tds[1]
city = list(city_td.stripped_strings)[0]
temp_td = tds[-2] # 最低气温
min_temp = list(temp_td.stripped_strings)[0]
ALL_DATA.append({'city':city, 'min_temp':int(min_temp)})
print({'city':city, 'min_temp':int(min_temp)})
def main():
urls = [
'http://www.weather.com.cn/textFC/hb.shtml',
'http://www.weather.com.cn/textFC/db.shtml',
'http://www.weather.com.cn/textFC/hd.shtml',
'http://www.weather.com.cn/textFC/hz.shtml',
'http://www.weather.com.cn/textFC/hn.shtml',
'http://www.weather.com.cn/textFC/xb.shtml',
'http://www.weather.com.cn/textFC/xn.shtml',
'http://www.weather.com.cn/textFC/gat.shtml'
]
for url in urls:
parse_page(url)
# 分析数据
# 根据最低气温进行排序
ALL_DATA.sort(key = lambda data: data['min_temp'])
data = ALL_DATA[0:10]
cities = list(map(lambda x: x['city'], data)) # 取出城市
temps = list(map(lambda x: x['min_temp'], data)) # 取出最低温度
# pyecharts
# pip install pyecharts
chart = Bar("中国天气最低气温排行榜")
chart.add('',cities,temps)
chart.render('temperature.html') # 渲染:生成html文件
if __name__ == '__main__':
main()
'''
# 字典排序测试:
ALL_DATA = [
{'city':'北京','min_temp':0},
{'city':'天津','min_temp':-8},
{'city':'石家庄','min_temp':-10}
]
# 对字典,按某一字段排序
ALL_DATA.sort(key = lambda data: data['min_temp'])
# 等同于以下方式:
# def sort_key(data):
# min_temp = data['min_temp']
# return min_temp
# ALL_DATA.sort(key=sort_key)
'''