Python爬取天气后报网2019年全国天气

爬天气

利用天气后报网爬取2019年全国城市天气，并提取各个城市的经纬度，并存入到MySQL中
代码如下:
import requests, re, pymysql
from lxml import etree

headers = {
“User-Agent”: “Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36”}
url = ‘http://www.tianqihoubao.com/lishi’
response = requests.get(url, headers=headers, timeout=60)
ele = etree.HTML(response.content.decode(‘gbk’))
a = ele.xpath(’/n//dd//@href’)
city = ele.xpath(’/n//dd//a//text()’)
c = -1

list = [
‘201901’] # ,‘201902’,‘201903’,‘201904’,‘201905’,‘201906’,‘201907’,‘201908’,‘201909’,‘201910’,‘201911’,‘201912’]

class dmysql:
def init(self, data=None, city=None, day_weather=None, night_weather=None, high_temp=None, low_temp=None,
day_wind=None, night_wind=None, lat=None, lng=None):
self.data = data
self.city = city
self.day_weather = day_weather
self.night_weather = night_weather
self.high_temperature = high_temp
self.low_temperature = low_temp
self.day_wind = day_wind
self.night_wind = night_wind
self.lat = lat
self.lng = lng

    self.connect = pymysql.connect(
        host='localhost',
        database='python1',
        port=3306,
        user='root',
        password='123456',
        charset='utf8',
        # use_unicode = False
    )
    self.cursor = self.connect.cursor()

# 保存数据到MySQL中
def save_mysql(self):
    sql = "insert into python1(date, city,day_weather,night_weather,high_temperature,low_temperature,day_wind,night_wind) VALUES(%s,%s,%s,%s,%s,%s,%s,%s)"
    self.cursor.execute(sql, (
        self.data, self.city, self.day_weather, self.night_weather, self.high_temperature, self.low_temperature,
        self.day_wind, self.night_wind))
    self.connect.commit()

def save_localtion(self):
    sql = "insert into python2(city,lat,lng) values(%s,%s,%s)"  # .format(self.city,self.lat,self.lng)
    self.cursor.execute(sql, (self.city, self.lat, self.lng))
    self.connect.commit()

for i in a:
c += 1
urld = ‘http://api.map.baidu.com/geocoder?output=json&key=f247cdb592eb43ebac6ccd27f796e2d2&address=’ + str(
city[c])
response = requests.get(urld, headers=headers, timeout=60)
answer = response.json()
lat = str(answer[‘result’][‘location’][‘lat’]) # 纬度
lng = str(answer[‘result’][‘location’][‘lng’]) # 经度
demo_localtion = dmysql(city=city[c], lat=lat, lng=lng)
demo_localtion.save_localtion()
for j in list:
na = ‘http://www.tianqihoubao.com’ + i[:-5] + ‘/month/’ + j + ‘.html’
res = requests.get(na, headers=headers, timeout=60)
ele = etree.HTML(res.content.decode(‘gbk’))
a1 = ele.xpath(’//tr//td//a//text()’)
b1 = ele.xpath(’//tr//td/text()’)
u1 = [re.sub(’\s’, ‘’, a1[i]) for i in range(len(a1))] # 日期
u2 = [re.sub(’\s’, ‘’, b1[i]) for i in range(len(b1))] # 天气情况
ull = [u2[i] for i in range(6, len(u2), 5)] # 天气状况
ulk = [u2[i] for i in range(7, len(u2), 5)] # 温度
ulj = [u2[i] for i in range(8, len(u2), 5)] # 风向
ulls = [ull[i].split(’/’) for i in range(len(ull))] # 天气状况切片
day_weather = [ulls[i][0] for i in range(len(ulls))] # 白天天气
night_weather = [ulls[i][1] for i in range(len(ulls))] # 晚上天气
ulks = [ulk[i].split(’/’) for i in range(len(ulk))] # 温度切片
high_temp = [ulks[i][0] for i in range(len(ulks))] # 最高温度
low_temp = [ulks[i][1] for i in range(len(ulks))] # 最低温度
uljs = [ulj[i].split(’/’) for i in range(len(ulj))] # 风向切片
day_wind = [uljs[i][0] for i in range(len(uljs))] # 白天风向
night_wind = [uljs[i][1] for i in range(len(uljs))] # 晚上风向
for k in range(len(u1)):
demo = dmysql(data=u1[k], city=city[c], day_weather=day_weather[k], night_weather=night_weather[k],
high_temp=high_temp[k], low_temp=low_temp[k], day_wind=day_wind[k],
night_wind=night_wind[k])
demo.save_mysql( )

MySQL下载链接www.mysql.com
建议下载5.7版本的msi程序包，安装过程可自行搜索
安装成功后，创建库和表
#创建库
create database python1;
#使用库
use python1;
#创建天气状况表
create table python1
(date varchar(30),
city varchar(30),
day_weather varchar(255),
night_weather varchar(255),
low_temperature varchar(30),
high_temperature varchar(30),
day_wind varchar(255),
night_wind varchar(255),
primary key (date,city));
#创建地理位置表
create table python2
(city varchar(30),
lat varchar(30),
lng varchar(30),
primary key (city)
);
之后运行pycharm代码即可。

结果展示: