【新手,代码水平请见谅,后期会优化】
数据网页链接:http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/
import urllib.request
from bs4 import BeautifulSoup
import time
import random
def url_open( url ):
formate = {
'User - Agent': 'Mozilla / 5.0(WindowsNT10.0;WOW64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 65.0.3325.181Safari / 537.36'
}
data = urllib.parse.urlencode(formate).encode(encoding='UTF8')
req = urllib.request.Request(url=url, data=data,method='POST')
response = urllib.request.urlopen(req)
html = response.read().decode('gbk')
return html
def bs(html, url):
# 从北京市开始
i = 0
soup = BeautifulSoup(html, "html.parser")
# 获取省
for li in soup.find_all('tr', class_="provincetr"):
for a in li:
if i > 0:
i = i - 1
else:
province = a.find('a').get_text()
# print(province)
# 获取省的下一级链接(市)
b = a.find('a')
plink = b.get('href')
# print(plink)
i = 1
# 获取市的相关数据
citylink = url_open(url + plink)
# print(citylink)
citysoup = BeautifulSoup(citylin