python爬虫爬取全国省市区

#之前写的代码

from requests_html import HTMLSession
import requests
import time
import re
import  datetime
import json
session = HTMLSession()
import mysql.connector
 
mydb = mysql.connector.connect(
    host="localhost",
    user="****",
    passwd="****",
    database="***"
)
mycursor = mydb.cursor()
 
sql = "TRUNCATE TABLE city_data"
mycursor.execute(sql)
 
mydb.commit()
 
# 获取省与直辖市
def get_Province_list():
    # 返回一个 response 对象
    response = session.get('http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/index.html')  # 单位秒数
 
    content = response.html.find('table.provincetable', first=True)
 
    li_list = content.find('a')
 
    for li in li_list:
        url = li.attrs['href']
        code=re.findall("\d+",url)[0]
        cname=li.text
        print(code)
        print('省级',cname)
       
 
        get_City_list(code)
 
      
# SQL 插入语句
 
        sql = "INSERT INTO city_data (name, code) VALUES (%s, %s)"
        val = (cname, code)
        mycursor.execute(sql, val)
 
        mydb.commit()  # 数据表内容有更新，必须使用到该语句
       
       
#获取市
def get_City_list(pid):
    # 返回一个 response 对象
    response = session.get('http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/'+pid+'.html')  # 单位秒数
 
    content = response.html.find('table.citytable', first=True)
    citys = content.find('tr.citytr')
    # print('citys',citys)
    # li_list = content.find('a')
 
    for city in citys:
        td_list = city.find('td')
        ycode=td_list[0].find('a')[0]
        # print('ycode', ycode)
        # url = li.attrs['href']
        # code=re.findall("\d+",url)[0]
        ccode=ycode.text[0:4]
        cname=td_list[1].find('a')[0].text
        print('ccode',ccode)
        print('城市名',cname)
      
        # SQL 插入语句
 
        sql = "INSERT INTO city_data (name, code,pId) VALUES (%s, %s, %s)"
        val = (cname, ccode,pid)
        mycursor.execute(sql, val)
 
        mydb.commit()  # 数据表内容有更新，必须使用到该语句
        #
     
        get_County_list(pid,ccode)
#获取县级市
def get_County_list(pid,cid):
    # 返回一个 response 对象
    response = session.get('http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/'+pid+'/'+cid+'.html')
 
    content = response.html.find('table.countytable', first=True)
    if content:
        citys = content.find('tr.countytr')
        # print('citys',citys)
        # li_list = content.find('a')
 
        for city in citys:
            td_list = city.find('td')
            ycode=td_list[0].find('a')
      
            if len(ycode) == 0:
                ccode =td_list[0].text
                cname = td_list[1].text
            else:
                ccode = ycode[0].text[0:6]
                cname = td_list[1].find('a')[0].text
       
 
                print('ccode',ccode)
                print('县级市',cname)
                sql = "INSERT INTO city_data (name, code,pId) VALUES (%s, %s, %s)"
                val = (cname, ccode,cid)
                mycursor.execute(sql, val)
 
                mydb.commit()  # 数据表内容有更新，必须使用到该语句
        else:
            print("没有循环数据!")
           
    else:
        print("表格不存在!")
 
if __name__ == '__main__':
    get_Province_list()
 

转载于:https://my.oschina.net/ytyjm/blog/3049563