python 爬虫获取国家统计局区划代码

最新推荐文章于 2024-09-12 10:14:29 发布

forever_insist

最新推荐文章于 2024-09-12 10:14:29 发布

阅读量1k

点赞数

文章标签： python 爬虫开发语言

本文链接：https://blog.csdn.net/forever_insist/article/details/128957198

版权

python 爬虫获取国家统计局区划代码

- 输出
- - 数据库

import pymysql
import requests
import lxml.etree as etree
import os


class ChineseArea(object):
    """获取省市区数据"""
    def __init__(self):
        """用户初始化数据库连接及定义存储数据的属性"""
        self.baseUrl = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2022/index.html'
        self.base = 'http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2022/'
        self.conn = pymysql.connect(host="127.0.0.1", port=3306, user="root", passwd="root", db="gov", charset='utf8')
        self.cur = self.conn.cursor()
        self.levels = {
            1: '//tr[@class="provincetr"]',
            2: '//tr[@class="citytr"]',
            3: '//tr[@class="countytr"]',
            4: '//tr[@class="towntr"]',
            5: '//tr[@class="villagetr"]'
        }

    def __del__(self):
        """关闭数据库连接"""
        if self.cur:
            self.cur.close()
        if self.conn:
            self.conn.close()

    def get_province_data(self, url):
        """爬行政区划代码公布页省级数据"""
        headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0',
                   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'}
        i = 0
        while i < 3:
            try:
                html = requests.get(url, headers=headers, timeout=20)
                html.encoding = 'utf8'  # 这里添加一行
                text = html.text
                return text
            except requests.exceptions.RequestException:
                i += 1
                print('超时' + url)

    def parse_province(self):
        """解析省页，返回list"""
        html = self.get_province_data(self.baseUrl)
        tree = etree.HTML(html, parser=etree.HTMLParser(encoding='gbk'))
        nodes = tree.xpath('//tr[@class="provincetr"]')
        values = []
        for node in nodes:
            items = node.xpath('./td')
            for item in items:
                value = {}
                next_url = item.xpath('./a/@href')
                province = item.xpath('./a/text()')
                if province:
                    print(province)
                    value['url'] = self.base + "".join(next_url)
                    value['name'] &#