python爬取全国真实地址_python爬虫爬取全国省市区

#之前写的代码

from requests_html import HTMLSession

import requests

import time

import re

import datetime

import json

session = HTMLSession()

import mysql.connector

mydb = mysql.connector.connect(

host="localhost",

user="****",

passwd="****",

database="***"

)

mycursor = mydb.cursor()

sql = "TRUNCATE TABLE city_data"

mycursor.execute(sql)

mydb.commit()

# 获取省与直辖市

def get_Province_list():

# 返回一个 response 对象

response = session.get('http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/index.html') # 单位秒数

content = response.html.find('table.provincetable', first=True)

li_list = content.find('a')

for li in li_list:

url = li.attrs['href']

code=re.findall("\d+",url)[0]

cname=li.text

print(code)

print('省级',cname)

get_City_list(code)

# SQL 插入语句

sql = "INSERT INTO city_data (name, code) VALUES (%s, %s)"

val = (cname, code)

mycursor.execute(sql, val)

mydb.commit() # 数据表内容有更新，必须使用到该语句

#获取市

def get_City_list(pid):

# 返回一个 response 对象

response = session.get('http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/'+pid+'.html') # 单位秒数

content = response.html.find('table.citytable', first=True)

citys = content.find('tr.citytr')

# print('citys',citys)

# li_list = content.find('a')

for city in citys:

td_list = city.find('td')

ycode=td_list[0].find('a')[0]

# print('ycode', ycode)

# url = li.attrs['href']

# code=re.findall("\d+",url)[0]

ccode=ycode.text[0:4]

cname=td_list[1].find('a')[0].text

print('ccode',ccode)

print('城市名',cname)

# SQL 插入语句

sql = "INSERT INTO city_data (name, code,pId) VALUES (%s, %s, %s)"

val = (cname, ccode,pid)

mycursor.execute(sql, val)

mydb.commit() # 数据表内容有更新，必须使用到该语句

#

get_County_list(pid,ccode)

#获取县级市

def get_County_list(pid,cid):

# 返回一个 response 对象

response = session.get('http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/'+pid+'/'+cid+'.html')

content = response.html.find('table.countytable', first=True)

if content:

citys = content.find('tr.countytr')

# print('citys',citys)

# li_list = content.find('a')

for city in citys:

td_list = city.find('td')

ycode=td_list[0].find('a')

if len(ycode) == 0:

ccode =td_list[0].text

cname = td_list[1].text

else:

ccode = ycode[0].text[0:6]

cname = td_list[1].find('a')[0].text

print('ccode',ccode)

print('县级市',cname)

sql = "INSERT INTO city_data (name, code,pId) VALUES (%s, %s, %s)"

val = (cname, ccode,cid)

mycursor.execute(sql, val)

mydb.commit() # 数据表内容有更新，必须使用到该语句

else:

print("没有循环数据!")

else:

print("表格不存在!")

if __name__ == '__main__':

get_Province_list()