使用爬虫(lxml)+requests收集厦门2016-2020每一天的历史天气信息
get_month_data
- 函数作用:得到每个月的数据集合,并返回一个pd.DataFrame。
- 其中使用了xpath的方式提取数据
etree_html = etree.HTML(html.text)
content = etree_html.xpath(Xpath)
- 使用numpy对数据reshape
- 将数据转换到DataFrame中
get_month_html
- 函数作用:得到每个月的链接集合。
- 通过几次尝试,发现该网站的命名规律是https://lishi.tianqi.com/城市拼音/年份月份.html(例如:https://lishi.tianqi.com/xiamen/201606.html)
- 简单的一个for循环搞定
完整代码
import requests
import numpy as np
import pandas as pd
from lxml import etree
def get_month_data(html, xpath, xpath_num):
df = pd.DataFrame([])
etree_html = etree.HTML(html.text)
for i, Xpath in enumerate(xpath):
content = etree_html.xpath(Xpath)
content = np.array(content)
content = content.reshape((len(content)//xpath_num[i],xpath_num[i]))
df[content[0]] = content[1:]
print(df)
return df
def get_month_html(link_base, start_year, end_year):
link_collection=[]
for year in range(start_year, end_year+1):
for month in range(1,13):
if month < 10:
link = link_base+str(year)+'0'+str(month)
else:
link = link_base+str(year)+str(month)
link+='.html'
link_collection.append(link)
return link_collection
if __name__ == '__main__':
start_year = 2016
end_year = 2020
link_base = 'https://lishi.tianqi.com/xiamen/'
xpath = ["//div[@class='th200']/text()","//div[@class='th140']/text()"]
xpath_num = [1,4]
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
link_collection = get_month_html(link_base,start_year,end_year)
weather_collection = pd.DataFrame([])
for link in link_collection:
html = requests.get(url=link, headers=headers)
df = get_month_data(html, xpath,xpath_num)
weather_collection = weather_collection.append(df,ignore_index=True)
weather_collection.to_csv('./2016-2020天气汇总.csv', encoding='gbk', index=False)