使用python抓取5级行政区划

大部分源码来自于：https://blog.csdn.net/MG_ApinG/article/details/82992913，谢谢作者

首先需要安装requests模块：

Windows安装： pip install requests
Linux安装：pip install requests

源码如下：

# -*- coding: GBK -*-
# author：zjp
# author：fudl 2019/09/13

"""
原版来自于网络：https://blog.csdn.net/MG_ApinG/article/details/82992913
通过国家统计局官网获取中国2018年所有城市数据
http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/

抓取后的数据格式：
	行政区划代码	上级行政区划代码	行政区划名称	行政区划全路径		级别（国为1级）
	110106000000	110100000000			丰台区		北京市|市辖区|丰台区	4
"""
import re
import requests
import time
import sys

fileSavePath = 'E://data/China_Province_2019.txt'  # 数据储存路径
fileSavePath2 = 'E://data/China_Province_2019_mistake.txt'  # 错误信息储存路径

code1 = []
results1 = []

code2 = []
results2 = []

code3 = []
results3 = []

code4 = []
results4 = []

code5 = []
results5 = []

Dates1 = []
code2Map = dict()
code3Map = dict()
code4Map = dict()
code5Map = dict()

n = 0
# 获取一级城市信息
url = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/index.html'
response = requests.get(url)
response.raise_for_status()  # 如果 HTTP 请求返回了不成功的状态码,Response.raise_for_status() 会抛出一个 HTTPError异常
response.encoding = "GBK"  # response.apparent_encoding从内容中分析出的响应内容编码方式
pattern = re.compile("<a href='(.*?)'>(.*?)<")
responseResult = list(set(re.findall(pattern, response.text)))  # 从首层页面获取进入第二层页面的html
# print('result1')

codes = []
codesNameDist = dict()
result1 = []

# 获取所有省份及代码
for result in responseResult:
    code = result[0].split('.')[0]
    codesNameDist[code] = result
    codes.append(code)

# 排序省份
codes.sort()

# 按照顺序抓取
for code in codes:
    result1.append(codesNameDist.get(code))

# 从一级城市获取二级城市信息
for cycle1 in range(len(result1)):
    try:
        address1_1 = result1[cycle1][0]
        code = address1_1.split(".")[0]

        # 如果上次卡住了重新运行时从上次卡住的省份开始，已经抓取的省份就不再抓取了，如下，已经抓取了北京市就<12
        if int(code) < 12:
            continue

        code1 = address1_1.split(".")[0] + '0000000000'
        address1 = result1[cycle1][1]  # 一级城市

        print('开始抓取-------->：', address1)

        # 1级到省
        with open(fileSavePath, 'a', encoding='GBK')as f:
            f.write(code1)
            f.write('\t')
            f.write('100000000000')
            f.write('\t')
            f.write(address1)
            f.write('\t')
            f.write(address1)
            f.write('\t')
            f.write('2')
            f.write('\n')
            f.close()

        url1 = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/{0}'.format(result1[cycle1][0])  # 一级城市url
        response1 = requests.get(url1)
        response1.raise_for_status()
        response1.encoding = "GBK"
        response1.close()
        pattern1 = re.compile("<a href='(.*?)'>(.*?)<")  # 正则表达式提取目标字段
        result2_1 = list(set(re.findall(pattern1, response1.text)))
        result2 = []

        # print("result2_1", result2_1)
        for result in result2_1:  # 爬取的城市信息和城市代码混在一起，需要将代码清除
            if '0' not in result[1]:
                result2.append(result)
            else:
                code2Map[result[0]] = result[1]

    except:
        print("Unexpected error:", sys.exc_info())
        with open(fileSavePath, 'a', encoding='GBK')as f:
            f.write('{0}|一级错误|一级错误|一级错误|{1}\n'.format('xd', sys.exc_info()))
            f.close()
        time.sleep(10)
        continue
    # 从二级城市获取三级城市信息
    for cycle2 in range(len(result2)):
        try:
            address2_1 = result2[cycle2][0]  # 二级城市
            address2 = result2[cycle2][1]  # 二级城市
            code2 = code2Map.get(result2[cycle2][0])

            # 2级到市
            address = '{0}|{1}'.format(address1, address2)
            with open(fileSavePath, 'a', encoding='GBK')as f:
                f.write(code2)
                f.write('\t')
                f.write(code1)
                f.write('\t')
                f.write(address2)
                f.write('\t')
                f.write(address)
                f.write('\t')
                f.write('3')
                f.write('\n')
                f.close()

            url2 = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/{0}'.format(result2[cycle2][0])  # 二级城市url
            # print(url2)
            response2 = requests.get(url2)
            response2.raise_for_status()
            response2.encoding = "GBK"
            response2.close()
            pattern2 = re.compile("<a href='(.*?)'>(.*?)<")
            result3_1 = list(set(re.findall(pattern2, response2.text)))
            result3 = []
            for result in result3_1:
                if '0' not in result[1]:
                    result3.append(result)
                else:
                    code3Map[result[0]] = result[1]
        except:
            print("Unexpected error:", sys.exc_info())
            with open(fileSavePath, 'a', encoding='GBK') as f:
                f.write('{0}|二级错误|二级错误|二级错误|{1}\n'.format(address1, sys.exc_info()))
                f.close()
            time.sleep(10)
            continue
        # 从三级城市获取四级城市信息
        for cycle3 in range(len(result3)):
            try:
                code3 = code3Map.get(result3[cycle3][0])
                address3 = result3[cycle3][1]  # 三级城市

                # 3级到区、县
                address = '{0}|{1}|{2}'.format(address1, address2, address3)
                with open(fileSavePath, 'a', encoding='GBK')as f:
                    f.write(code3)
                    f.write('\t')
                    f.write(code2)
                    f.write('\t')
                    f.write(address3)
                    f.write('\t')
                    f.write(address)
                    f.write('\t')
                    f.write('4')
                    f.write('\n')
                    f.close()

                url3 = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/{0}/{1}' \
                    .format(result3[cycle3][0][3:5], result3[cycle3][0])  # 三级城市url
                response3 = requests.get(url3)
                response3.raise_for_status()
                response3.encoding = "GBK"
                response3.close()
                pattern3 = re.compile("<a href='(.*?)'>(.*?)<")
                result4_1 = list(set(re.findall(pattern3, response3.text)))
                result4 = []
                for result in result4_1:
                    if '0' not in result[1]:
                        result4.append(result)
                    else:
                        code4Map[result[0]] = result[1]
            except:
                print("Unexpected error:", sys.exc_info())
                with open(fileSavePath, 'a', encoding='GBK')as f:
                    f.write('{0}|三级错误|三级错误|三级错误|{1}\n'.format(address2, sys.exc_info()))
                    f.close()
                time.sleep(10)
                continue
            # 从四级城市获取四五级城市信息
            for cycle4 in range(len(result4)):
                try:
                    code4 = code4Map.get(result4[cycle4][0])
                    address4 = result4[cycle4][1]  # 四级城市

                    # 4级到镇、街道
                    address = '{0}|{1}|{2}|{3}'.format(address1, address2, address3, address4)
                    with open(fileSavePath, 'a', encoding='GBK')as f:
                        f.write(code4)
                        f.write('\t')
                        f.write(code3)
                        f.write('\t')
                        f.write(address4)
                        f.write('\t')
                        f.write(address)
                        f.write('\t')
                        f.write('5')
                        f.write('\n')
                        f.close()

                    url4 = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2018/{0}/{1}/{2}' \
                        .format(result4[cycle4][0][3:5], result4[cycle4][0][5:7], result4[cycle4][0])  # 四级城市url
                    response4 = requests.get(url4)
                    response4.raise_for_status()
                    response4.encoding = "GBK"
                    response4.close()
                    time.sleep(1)
                    pattern4_1 = re.compile("<a href='(.*?)'>(.*?)<")
                    # 可能已经到最后一层
                    pattern4 = re.compile("<tr class='villagetr'><td>(.*?)</td><td>.*?</td><td>(.*?)</td></tr>")
                    result5_1 = list(set(re.findall(pattern4, response4.text)))
                    result5 = []
                    for result in result5_1:
                        if '0' not in result[1]:
                            result5.append(result[1])
                        code5Map[result[1]] = result[0]

                    if result5:
                        for cycle5 in result5:
                            # 5级到村
                            address = '{0}|{1}|{2}|{3}|{4}'.format(address1, address2, address3, address4, cycle5)
                            code5 = code5Map[cycle5]
                            with open(fileSavePath, 'a', encoding='GBK')as f:
                                f.write(code5)
                                f.write('\t')
                                f.write(code4)
                                f.write('\t')
                                f.write(cycle5)
                                f.write('\t')
                                f.write(address)
                                f.write('\t')
                                f.write('6')
                                f.write('\n')
                                f.close()
                    else:
                        print('此处第五层城市结果为空,网址为: {0}'.format(url4))
                        with open(fileSavePath2, 'a', encoding='GBK')as f2:
                            f2.write('此处第五层城市结果为空,网址为: {0}'.format(url4))
                            f2.close()

                except:
                    print("Unexpected error:", sys.exc_info())
                    with open(fileSavePath, 'a', encoding='GBK')as f:
                        f.write('{0}|四级错误|四级错误|四级错误|{1}\n'.format(address3, sys.exc_info()))
                        f.close()
                    time.sleep(10)
                    continue
    print('抓取结束==============>：', address1)
print('well_done')