爬天气历史数据
[http://lishi.tianqi.com/chongqing/index.html]
一、使用requests+bs4采取数据
1、获取URL连接,URL分两部分,城市名称和年月,可以使用数据遍历月份,请求URL,解析response。
def get_url(cityname,start_year,end_year):
years = list(range(start_year, end_year))
months = list(range(1, 13))
suburl = 'http://www.tianqihoubao.com/lishi/'
urllist = []
for year in years:
for month in months:
if month < 10:
url = suburl + cityname + '/month/'+ str(year) + (str(0) + str(month)) + '.html'
else:
url = suburl + cityname + '/month/' + str(year) + str(month) + '.html'
urllist.append(url.strip())
return urllist
2、我们在上面使用了cityname,而cityname就是我们需要抓取的城市的城市名称,我们需要查询数据库获取城市名称,遍历它,将城市名称和开始年份,结束年份,给上面的函数。
def get_cityid(db_conn,db_cur,url):
suburl = url.split('/')
sql = 'select cityid from city where cityname = %s '
db_cur.execute(sql,suburl[4])
cityid = db_cur.fetchone()
idlist = list(cityid)
return idlist[0]
3、用bs解析网页代码
def parse_html_bs(db_conn,db_cur,url):
proxy = get_proxy()
proxies = {
'http': 'http://' + proxy,
'https': 'https://' + proxy,
}
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
'Connection': 'close'
}
# 获取天气数据的html网页源代码
weather_data = requests.get(url=url, headers=headers,proxies = proxies).text
weather_data_new =(weather_data.replace('\n','').replace('\r','').replace(' ',''))
soup = BeautifulSoup(weather_data_new,'lxml')
table = soup.find_all(['td'])
# 获取城市id
cityid = get_cityid(db_conn, db_cur, url)
listall = []
for t in list(table):
ts = t.string
listall.append(ts)
n= 4
sublist = [listall[i:i+n] for i in range(0,len(listall),n)]
sublist.remove(sublist[0])
flist = []
# 将列表元素中的最高和最低气温拆分,方便后续数据分析,并插入城市代码
for sub in sublist:
if sub == sublist[0]:
pass
sub2 = sub[2].split('/')
sub.remove(sub[2])
sub.insert(2, sub2[0])
sub.insert(3, sub2[1])
sub.insert(0, cityid) # 插入城市代码
flist.append(sub)
return flist
二、用scrapy框架采集天气
1、在spiders文件中编写爬虫
# -*- coding: utf-8 -*-
import scrapy
from bs4 import BeautifulSoup
from scrapy import Request
from lxml import etree
from ..items import TianqiItem
class WeatherSpider(scrapy.Spider):
name = 'weather'
# allowed_domains = ['tianqihoubao']
start_urls = ['https://lishi.tianqi.com/']
def parse(self, response):
soup = BeautifulSoup(response.text, 'lxml')
citylists = soup.find_all(name='div', class_='box-base')
for citys in citylists:
for city in citys.find_all(name='li'):
url = 'https://lishi.tianqi.com' + city.a['href']
yield Request(url=url,
callback=self.parse_citylist)
def parse_citylist(self, response):
soup = BeautifulSoup(response.text, 'lxml')
monthlist = soup.find_all(name='div', class_='tqtongji1')
for months in monthlist:
for month in months.find_all(name='li'):
if month.a['href'].endswith('01.html'):
url = month.a['href']
url = 'https://lishi.tianqi.com' + url
yield Request(url=url, callback=self.parse_weather)
def parse_weather(self, response): # 解析网页数据,返回数据给pipeline处
url = response.url
cityname = url.split('/')[4]
weather_html = etree.HTML(response.text)
table = weather_html.xpath('//table//tr//td//text()')
listall = []
for t in table:
if t.strip() == '':
continue
t1 = t.replace(' ', '')
t2 = t1.replace('\r\n', '')
listall.append(t2.strip())
n = 4
sublist = [listall[i:i + n] for i in range(0, len(listall), n)]
# 删除表头第一行
sublist.remove(sublist[0])
for sub in sublist:
if sub == sublist[0]:
pass
sub2 = sub[2].split('/')
sub.remove(sub[2])
sub.insert(2, sub2[0])
sub.insert(3, sub2[1])
sub.insert(0, cityname)
Weather = TianqiItem()
Weather['cityname'] = sub[0]
Weather['data'] = sub[1]
Weather['tq'] = sub[2]
Weather['maxtemp'] = sub[3]
Weather['mintemp'] = sub[4]
Weather['fengli'] = sub[5]
yield Weather