1 课题介绍
全球Covid-19大危机影响我们的生活,我们的出行、交流、教育、经济等都发生了巨大的变化,全球疫情大数据可视化分析与展示,可用于社会各界接入疫情数据,感知疫情相关情况的实时交互式态势,是重要的疫情分析、防控决策依据。
我国爆发的疫情,对我们的日常生活带来了极大的影响,疫情严重期间,大家都谈“疫”色变,大家对于了解疫情的情况具有巨大的需求;
代码在最下面(要的请私)
Python+SpringBoot+Vue
2 运行效果
3 关键代码
3.1 数据爬虫
疫情数据爬虫,就是给网站发起请求,并从响应中提取需要的数据
1、发起请求,获取响应
- 通过http库,对目标站点进行请求。等同于自己打开浏览器输入网址
- 常用库:urllib、requests
- 服务器会返回请求的内容一般为:HTML、文档、JSON字符串等
2、解析内容
- 寻找自己需要的信息,也就是利用正则表达式或者其他库提取目标信息
- 常用库:re、beautifulsoup4
3、保存数据
- 将解析到的数据持久化到数据库中
import pymysql import time import json import traceback #追踪异常 import requests def get_tencent_data(): """ :return: 返回历史数据和当日详细数据 """ url = '' url_his='' #最基本的反爬虫 headers = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36', } r = requests.get(url, headers) #使用requests请求 res = json.loads(r.text) # json字符串转字典 data_all = json.loads(res['data'])
<span class="token comment">#再加上history的配套东西</span> r_his<span class="token operator">=</span>requests<span class="token punctuation">.</span>get<span class="token punctuation">(</span>url_his<span class="token punctuation">,</span>headers<span class="token punctuation">)</span> res_his<span class="token operator">=</span>json<span class="token punctuation">.</span>loads<span class="token punctuation">(</span>r_his<span class="token punctuation">.</span>text<span class="token punctuation">)</span> data_his<span class="token operator">=</span>json<span class="token punctuation">.</span>loads<span class="token punctuation">(</span>res_his<span class="token punctuation">[</span><span class="token string">'data'</span><span class="token punctuation">]</span><span class="token punctuation">)</span> history <span class="token operator">=</span> <span class="token punctuation">{<!-- --></span><span class="token punctuation">}</span> <span class="token comment"># 历史数据</span> <span class="token keyword">for</span> i <span class="token keyword">in</span> data_his<span class="token punctuation">[</span><span class="token string">"chinaDayList"</span><span class="token punctuation">]</span><span class="token punctuation">:</span> ds <span class="token operator">=</span> <span class="token string">"2020."</span> <span class="token operator">+</span> i<span class="token punctuation">[</span><span class="token string">"date"</span><span class="token punctuation">]</span> tup <span class="token operator">=</span> time<span class="token punctuation">.</span>strptime<span class="token punctuation">(</span>ds<span class="token punctuation">,</span> <span class="token string">"%Y.%m.%d"</span><span class="token punctuation">)</span> ds <span class="token operator">=</span> time<span class="token punctuation">.</span>strftime<span class="token punctuation">(</span><span class="token string">"%Y-%m-%d"</span><span class="token punctuation">,</span> tup<span class="token punctuation">)</span> <span class="token comment"># 改变时间格式,不然插入数据库会报错,数据库是datetime类型</span> confirm <span class="token operator">=</span> i<span class="token punctuation">[</span><span class="token string">"confirm"</span><span class="token punctuation">]</span> suspect <span class="token operator">=</span> i<span class="token punctuation">[</span><span class="token string">"suspect"</span><span class="token punctuation">]</span> heal <span class="token operator">=</span> i<span class="token punctuation">[</span><span class="token string">"heal"</span><span class="token punctuation">]</span> dead <span class="token operator">=</span> i<span class="token punctuation">[</span><span class="token string">"dead"</span><span class="token punctuation">]</span> history<span class="token punctuation">[</span>ds<span class="token punctuation">]</span> <span class="token operator">=</span> <span class="token punctuation">{<!-- --></span><span class="token string">"confirm"</span><span class="token punctuation">:</span> confirm<span class="token punctuation">,</span> <span class="token string">"suspect"</span><span class="token punctuation">:</span> suspect<span class="token punctuation">,</span> <span class="token string">"heal"</span><span class="token punctuation">:</span> heal<span class="token punctuation">,</span> <span class="token string">"dead"</span><span class="token punctuation">:</span> dead<span class="token punctuation">}</span> <span class="token keyword">for</span> i <span class="token keyword">in</span> data_his<span class="token punctuation">[</span><span class="token string">"chinaDayAddList"</span><span class="token punctuation">]</span><span class="token punctuation">:</span> ds <span class="token operator">=</span> <span class="token string">"2020."</span> <span class="token operator">+</span> i<span class="token punctuation">[</span><span class="token string">"date"</span><span class="token punctuation">]</span> tup <span class="token operator">=</span> time<span class="token punctuation">.</span>strptime<span class="token punctuation">(</span>ds<span class="token punctuation">,</span> <span class="token string">"%Y.%m.%d"</span><span class="token punctuation">)</span> ds <span class="token operator">=</span> time<span class="token punctuation">.</span>strftime<span class="token punctuation">(</span><span class="token string">"%Y-%m-%d"</span><span class="token punctuation">,</span> tup<span class="token punctuation">)</span> confirm <span class="token operator">=</span> i<span class="token punctuation">[</span><span class="token string">"confirm"</span><span class="token punctuation">]</span> suspect <span class="token operator">=</span> i<span class="token punctuation">[</span><span class="token string">"suspect"</span><span class="token punctuation">]</span> heal <span class="token operator">=</span> i<span class="token punctuation">[</span><span class="token string">"heal"</span><span class="token punctuation">]</span> dead <span class="token operator">=</span> i<span class="token punctuation">[</span><span class="token string">"dead"</span><span class="token punctuation">]</span> history<span class="token punctuation">[</span>ds<span class="token punctuation">]</span><span class="token punctuation">.</span>update<span class="token punctuation">(</span><span class="token punctuation">{<!-- --></span><span class="token string">"confirm_add"</span><span class="token punctuation">:</span> confirm<span class="token punctuation">,</span> <span class="token string">"suspect_add"</span><span class="token punctuation">:</span> suspect<span class="token punctuation">,</span> <span class="token string">"heal_add"</span><span class="token punctuation">:</span> heal<span class="token punctuation">,</span> <span class="token string">"dead_add"</span><span class="token punctuation">:</span> dead<span class="token punctuation">}</span><span class="token punctuation">)</span> details <span class="token operator">=</span> <span class="token punctuation">[</span><span class="token punctuation">]</span> <span class="token comment"># 当日详细数据</span> update_time <span class="token operator">=</span> data_all<span class="token punctuation">[</span><span class="token string">"lastUpdateTime"</span><span class="token punctuation">]</span> data_country <span class="token operator">=</span> data_all<span class="token punctuation">[</span><span class="token string">"areaTree"</span><span class="token punctuation">]</span> <span class="token comment"># list 25个国家</span> data_province <span class="token operator">=</span> data_country<span class="token punctuation">[</span><span class="token number">0</span><span class="token punctuation">]</span><span class="token punctuation">[</span><span class="token string">"children"</span><span class="token punctuation">]</span> <span class="token comment"># 中国各省</span> <span class="token keyword">for</span> pro_infos <span class="token keyword">in</span> data_province<span class="token punctuation">:</span> province <span class="token operator">=</span> pro_infos<span class="token punctuation">[</span><span class="token string">"name"</span><span class="token punctuation">]</span> <span class="token comment"># 省名</span> <span class="token keyword">for</span> city_infos <span class="token keyword">in</span> pro_infos<span class="token punctuation">[</span><span class="token string">"children"</span><span class="token punctuation">]</span><span class="token punctuation">:</span> city <span class="token operator">=</span> city_infos<span class="token punctuation">[</span><span class="token string">"name"</span><span class="token punctuation">]</span> confirm <span class="token operator">=</span> city_infos<span class="token punctuation">[</span><span class="token string">"total"</span><span class="token punctuation">]</span><span class="token punctuation">[</span><span class="token string">"confirm"</span><span class="token punctuation">]</span> confirm_add <span class="token operator">=</span> city_infos<span class="token punctuation">[</span><span class="token string">"today"</span><span class="token punctuation">]</span><span class="token punctuation">[</span><span class="token string">"confirm"</span><span class="token punctuation">]</span> heal <span class="token operator">=</span> city_infos<span class="token punctuation">[</span><span class="token string">"total"</span><span class="token punctuation">]</span><span class="token punctuation">[</span><span class="token string">"heal"</span><span class="token punctuation">]</span> dead <span class="token operator">=</span> city_infos<span class="token punctuation">[</span><span class="token string">"total"</span><span class="token punctuation">]</span><span class="token punctuation">[</span><span class="token string">"dead"</span><span class="token punctuation">]</span> details<span class="token punctuation">.</span>append<span class="token punctuation">(</span><span class="token punctuation">[</span>update_time<span class="token punctuation">,</span> province<span class="token punctuation">,</span> city<span class="token punctuation">,</span> confirm<span class="token punctuation">,</span> confirm_add<span class="token punctuation">,</span> heal<span class="token punctuation">,</span> dead<span class="token punctuation">]</span><span class="token punctuation">)</span> <span class="token keyword">return</span> history<span class="token punctuation">,</span> details
数据表结构
history表存储每日的总数据
CREATE TABLE history ( ds datetime NOT NULL COMMENT ‘日期’, confirm int(11) DEFAULT NULL COMMENT ‘累计确诊’, confirm_add int(11) DEFAULT NULL COMMENT ‘当日新增确诊’, suspect int(11) DEFAULT NULL COMMENT ‘剩余疑似’, suspect_add int(11) DEFAULT NULL COMMENT ‘当日新增疑似’, heal int(11) DEFAULT NULL COMMENT ‘累计治愈’, heal_add int(11) DEFAULT NULL COMMENT ‘当日新增治愈’, dead int(11) DEFAULT NULL COMMENT ‘累计死亡’, dead_add int(11) DEFAULT NULL COMMENT ‘当日新增死亡’, PRIMARY KEY (ds) USING BTREE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
details表存储每日的详细数据
CREATE TABLE details ( id int(11) NOT NULL AUTO_INCREMENT, update_time datetime DEFAULT NULL COMMENT ‘数据最后更新时间’, province varchar(50) DEFAULT NULL COMMENT ‘省’, city varchar(50) DEFAULT NULL COMMENT ‘市’, confirm int(11) DEFAULT NULL COMMENT ‘累计确诊’, confirm_add int(11) DEFAULT NULL COMMENT ‘新增确诊’, heal int(11) DEFAULT NULL COMMENT ‘累计治愈’, dead int(11) DEFAULT NULL COMMENT ‘累计死亡’, PRIMARY KEY (id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
整体的数据库图表:
3.2 可视化部分
echarts绘制图表
def get_c1_data(): """ :return: 返回大屏div id=c1 的数据 """ # 因为会更新多次数据,取时间戳最新的那组数据 sql = "select sum(confirm)," \ "(select suspect from history order by ds desc limit 1)," \ "sum(heal)," \ "sum(dead) " \ "from details " \ "where update_time=(select update_time from details order by update_time desc limit 1) " res = query(sql) res_list = [str(i) for i in res[0]] res_tuple=tuple(res_list) return res_tuple
中国疫情地图实现
def get_c2_data(): """ :return: 返回各省数据 """ # 因为会更新多次数据,取时间戳最新的那组数据 sql = "select province,sum(confirm) from details " \ "where update_time=(select update_time from details " \ "order by update_time desc limit 1) " \ "group by province" res = query(sql) return res
全国累计趋势
def get_l1_data():
"""
:return:返回每天历史累计数据
"""
sql = "select ds,confirm,suspect,heal,dead from history"
res = query(sql)
return res
def get_l2_data():
“”"
:return:返回每天新增确诊和疑似数据
“”"
sql = “select ds,confirm_add,suspect_add from history”
res = query(sql)
return res
def get_r1_data():
“”"
:return: 返回非湖北地区城市确诊人数前5名
“”"
sql = 'SELECT city,confirm FROM ‘
’(select city,confirm from details '
'where update_time=(select update_time from details order by update_time desc limit 1) '
'and province not in (“湖北”,“北京”,“上海”,“天津”,“重庆”) '
'union all '
'select province as city,sum(confirm) as confirm from details '
'where update_time=(select update_time from details order by update_time desc limit 1) '
'and province in (“北京”,“上海”,“天津”,“重庆”) group by province) as a '
‘ORDER BY confirm DESC LIMIT 5’
res = query(sql)
return res
疫情热搜
def get_r2_data(): """ :return: 返回最近的20条热搜 """ sql = 'select content from hotsearch order by id desc limit 20' res = query(sql) # 格式 (('民警抗疫一线奋战16天牺牲1037364',), ('四川再派两批医疗队1537382',) return re
2208446598