获取最新的行政区划分数据
使用python 获取
from mysql_handler import MysqlHandler
from mysql_handler_sub import MysqlHandlerSub
from html_downloader import HtmlDownloader
from html_parser import HtmlParser
import traceback
import json
class CodeSpider(object):
def __init__(self):
self.mysql_handler = MysqlHandler()
self.html_downloader = HtmlDownloader()
self.html_parser = HtmlParser()
self.root_url = 'http://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023/index.html'
self.split_url = 'http://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023/'
self.province_array = [11,12,13]
def craw(self):
self.get_Province_list()
def get_Province_list(self):
downloading_url = self.root_url
html_content = self.html_downloader.download(downloading_url)
self.province_url_list = self.html_parser.province_parser(html_content, self.split_url)
for province_name, province_url, province_code in self.province_url_list:
print( province_name, province_url, province_code)
area_code = str(province_code) + '000000000'
province_id = self.mysql_handler.insert(1, province_name, area_code,0,0)
self.get_City_list(province_code,area_code,province_id)
def get_City_list(self,code, parent_code, parent_id):
print('获取-市')
print(code, parent_code, parent_id)
downloading_url = self.split_url + code + '.html'
html_content = self.html_downloader.download(downloading_url)
self.city_url_list = self.html_parser.city_parser(html_content, self.split_url)
for city_name, city_url, city_code in self.city_url_list:
city_id = self.mysql_handler.insert(2, city_name,city_code,parent_code,parent_id)
if city_url is None:
continue
self.get_county_list(city_url,code, city_code, city_id)
def get_county_list(self,_url,code, parent_code, parent_id):
print('获取 - 县级市')
print(_url,code, parent_code, parent_id)
downloading_url = self.split_url + _url
html_content = self.html_downloader.download(downloading_url)
self.county_url_list = self.html_parser.county_parser(html_content, self.split_url + code + "/")
for county_name, county_url, county_code in self.county_url_list:
city_id = self.mysql_handler.insert(3, county_name,county_code,parent_code,parent_id)
if county_url is None:
continue
code1 = county_url.split('/')
self.get_town_list(county_url,code,code1[0],county_code,city_id)
def get_town_list(self,_url,code, code1,parent_code, parent_id):
print('获取乡镇/街道')
print(_url,code, code1, parent_id)
downloading_url = self.split_url +code+ '/' + _url
html_content = self.html_downloader.download(downloading_url)
self.town_url_list = self.html_parser.town_parser(html_content, self.split_url +code+ '/')
for town_name, town_url, town_code in self.town_url_list:
town_id = self.mysql_handler.insert( 4,town_name, town_code,parent_code,parent_id)
if town_url is None:
continue
self.get_village_list(town_url,code,code1,town_code,town_id)
def get_village_list(self,_url,code, code1,parent_code, parent_id):
print('获取村/居委会')
print(_url,code, code1, parent_id)
downloading_url =self.split_url +code+'/'+code1+'/'+_url
html_content = self.html_downloader.download(downloading_url)
self.village_url_list = self.html_parser.village_parser(html_content, self.split_url+"/")
for village_name, village_url, village_code in self.village_url_list:
self.mysql_handler.insert(5, village_name, village_code,parent_code,parent_id)
if __name__ == '__main__':
obj_spider = CodeSpider()
obj_spider.craw()