中国疫情实时数据爬取
import requests
import pandas as pd
from bs4 import BeautifulSoup
import re
url = 'https://ncov.dxy.cn/ncovh5/view/pneumonia?from=timeline&isappinstalled=0'
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'
}
provinceName_re = re.compile(r'"provinceName":"(.*?)",')
provinceShortName_re = re.compile(r'"provinceShortName":"(.*?)",')
currentConfirmedCount_re = re.compile(r'"currentConfirmedCount":(.*?),')
confirmedCount_re = re.compile(r'"confirmedCount":(.*?),')
suspectedCount_re = re.compile(r'"suspectedCount":(.*?),')
curedCount_re = re.compile(r'"curedCount":(.*?),')
deadCount_re = re.compile(r'"deadCount":(.*?),')
comment_re = re.compile(r'"comment":"(.*?)",')
locationId_re = re.compile(r'"locationId":(.*?),')
statisticsData_re = re.compile(r'"statisticsData":"(.*?)",')
cities_re = re.compile(r'"cities":\[\{(.*?)\}\]')
cityName_re = re.compile(r'"cityName":"(.*?)",')
currentConfirmedCount_1_re = re.compile(r'"currentConfirmedCount":(.*?),')
confirmedCount_1_re = re.compile(r'"confirmedCount":(.*?),')
suspectedCount_1_re = re.compile(r'"suspectedCount":(.*?),')
curedCount_1_re = re.compile(r'"curedCount":(.*?),')
deadCount_1_re = re.compile(r'"deadCount":(.*?),')
locationId_1_re = re.compile(r'"locationId":(.*?)\},')
datas = requests.get(url,headers = headers)
datas.encoding = 'utf-8'
soup = BeautifulSoup(datas.text,'lxml')
data = soup.find_all('script',{'id':'getAreaStat'})
data = str(data)
data_str = data[54:-23]
citiess = re.sub(cities_re,'8888',data_str)
provinceNames = re.findall(provinceName_re,citiess)
provinceShortNames = re.findall(provinceShortName_re,citiess)
currentConfirmedCounts = re.findall(currentConfirmedCount_re,citiess)
confirmedCounts = re.findall(confirmedCount_re,citiess)
suspectedCounts = re.findall(suspectedCount_re,citiess)
curedCounts = re.findall(curedCount_re,citiess)
deadCounts = re.findall(deadCount_re,citiess)
comments = re.findall(comment_re,citiess)
locationIds = re.findall(locationId_re,citiess)
statisticsDatas = re.findall(statisticsData_re,citiess)
citiess_str1 = re.findall(cities_re,data_str)
citiess_str = str(citiess_str1)
cityName = re.findall(cityName_re,citiess_str)
currentConfirmedCount_1 = re.findall(currentConfirmedCount_1_re,citiess_str)
confirmedCount_1 = re.findall(confirmedCount_1_re,citiess_str)
suspectedCount_1 = re.findall(suspectedCount_1_re,citiess_str)
curedCount_1 = re.findall(curedCount_1_re,citiess_str)
deadCount_1 = re.findall(deadCount_1_re,citiess_str)
df = {
'地区代码':pd.Series(locationIds),
'省':pd.Series(provinceNames),
'省区短名':pd.Series(provinceShortNames),
'当前确诊':pd.Series(currentConfirmedCounts),
'累计确诊':pd.Series(confirmedCounts),
'疑似确诊':pd.Series(suspectedCounts),
'治愈人数':pd.Series(curedCounts),
'死亡人数':pd.Series(deadCounts),
'评论':pd.Series(comments),
'统计数据区':pd.Series(statisticsDatas),
}
pds = pd.DataFrame(df)
df2 = {
'城市名':pd.Series(cityName),
'当前确诊':pd.Series(currentConfirmedCount_1),
'累计确诊':pd.Series(confirmedCount_1),
'疑似确诊':pd.Series(suspectedCount_1),
'治愈人数':pd.Series(curedCount_1),
'死亡人数':pd.Series(deadCount_1),
}
pdc = pd.DataFrame(df2)
pds.to_csv('templates\china_cor_data.csv',index=True)
pdc.to_csv('templates\cities_cor_data.csv',index=True)
df1 = pd.read_csv('templates\china_cor_data.csv')
df2 = pd.read_csv('templates\cities_cor_data.csv')
import numpy as np
data1 = df1[['省区短名','当前确诊','累计确诊','治愈人数']]
analysis1 = data1[['当前确诊','累计确诊','治愈人数']].describe().loc[['mean','std','min','max'],:]
analysis1.to_csv(r'templates\analysis1.csv',index=True)
data2 = df2[['城市名','当前确诊','累计确诊','治愈人数']]
analysis2 = data2[['当前确诊','累计确诊','治愈人数']].describe().loc[['mean','std','min','max'],:]
analysis2.to_csv(r'templates\analysis2.csv',index=True)
data2 = pds.loc[:, ['省区短名','当前确诊']]
df_list = data2.values.tolist()
from pyecharts import options as opts
from pyecharts.charts import Geo
from pyecharts.globals import ChartType
def china_map_1():
c = (
Geo()
.add_schema(maptype="china")
.add(
"geo",
df_list,
type_=ChartType.HEATMAP,
)
.set_series_opts(label_opts=opts.LabelOpts(is_show=False))
.set_global_opts(
visualmap_opts=opts.VisualMapOpts(),
title_opts=opts.TitleOpts(title="国内省份疫情实况"),
toolbox_opts=opts.ToolboxOpts(
is_show=True,
),
)
.render("templates\map_visualmap_china.html")
)
from pyecharts.charts import Bar
def analysis1_bar():
c = (
Bar()
.add_xaxis(['当前确诊','累计确诊','治愈人数'])
.add_yaxis("平均值",analysis1.loc['mean', :].values.tolist(), stack="stack1")
.add_yaxis("最大值",analysis1.loc['max',:].values.tolist(), stack="stack2")
.set_series_opts(
label_opts=opts.LabelOpts(is_show=False),
markline_opts=opts.MarkLineOpts(
data=[
opts.MarkLineItem(type_="max", name="最大值"),
opts.MarkLineItem(type_="average", name="平均值"),
]
),
)
.set_global_opts(title_opts=opts.TitleOpts(title="省级.简单统计结果"))
.render(r"templates\analysis1_bar_stack.html")
)
美国及全球疫情实时数据爬取
import requests
import urllib3
import re
from bs4 import BeautifulSoup
import csv
import pandas as pd
import numpy as np
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
"Referer": "http://m.sinovision.net",
'Host': 'm.sinovision.net',
}
url = 'http://m.sinovision.net/newpneumonia.php'
global datas
datas = []
def statistic(url):
response = requests.get(url, headers=headers)
data = response.content.decode('utf-8')
soup = BeautifulSoup(data, 'html5lib')
count = 0; global datas
for tag in soup.find_all('div', class_='main-block'):
country = tag.find('span', attrs={"class": "area"}).get_text()
cer = tag.find('span',class_='confirm').get_text()
death = tag.find('span',class_='dead').get_text()
cure = tag.find('span',class_='cured-notag')
if cure is None:
cure = 0
else:
cure = tag.find('span',class_='cured-notag').get_text()
data = {"国家": country, "确诊": cer, '死亡': death, '治愈':cure}
datas.append(data)
statistic(url)
for index,data in enumerate(datas):
if data['国家'] == 'US':
print(index)
else:
continue
US_datasets = datas[:56]
global_datasets = datas[57:]
name = ["国家", "确诊", "死亡", "治愈"]
usdata = pd.DataFrame(columns=["国家", "确诊", "死亡"], data=US_datasets)
usdata.to_csv(r'templates\usdata.csv')
globaldata = pd.DataFrame(columns=name, data=global_datasets)
globaldata.to_csv('templates\globaldata.csv')
df = pd.read_csv(r'templates\usdata.csv')
result2 = [
[
value['国家'],
value['确诊']
]
for index,value in df.iterrows()
]
result3 = [
[
value['国家'],
value['死亡']
]
for index,value in df.iterrows()
]
x_data = np.mean([item[1] for item in result2])
x_data2 = np.median([item[1] for item in result2])
y_data = np.mean([item[1] for item in result3])
y_data2 = np.median([item[1] for item in result3])
str1="平均确诊"+str(round(x_data,2))+'\n'+'\n'+"中位数"+str(round(x_data2,2))
str2="平均死亡"+str(round(y_data,2))+'\n'+'\n'+"中位数"+str(round(y_data2,2))
from pyecharts import options as opts
from pyecharts.charts import Map
from pyecharts.globals import ThemeType
from pyecharts.charts import Grid
import pyecharts.options as opts
from pyecharts.charts import Line
def qz():
c = (
Map(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
.add("确诊人数",result2, "美国")
.set_global_opts(
title_opts=opts.TitleOpts(title="美国确诊人数",subtitle=str1),
visualmap_opts=opts.VisualMapOpts(max_=4000000)
)
.render("templates\确诊人数.html")
)
def sw():
d = (
Map(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
.add("死亡人数",result3, "美国")
.set_global_opts(
title_opts=opts.TitleOpts(title="死亡人数",subtitle=str2),
visualmap_opts=opts.VisualMapOpts(max_=65000)
)
.render("templates\death.html")
)