对应的视频讲解地址请点击:python爬虫实例讲解_哔哩哔哩_bilibilipython爬虫教程,数据信息采集爬取https://www.bilibili.com/video/BV13L4y137Q5?p=1
#!/usr/bin/env python3
# coding:utf-8
# @Time: 2022/1/17
# 获取全国天气信息(城市和对应的最低气温,保存到excel,并生成柱形图)
import requests # pip install requests
import re
from lxml import etree
import pandas as pd
from pyecharts.charts import Bar
class Spider:
def __init__(self):
self.headers = {
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36"
}
self.content_list = []
# 获取网址
def get_url(self):
url = "http://www.weather.com.cn/textFC/hb.shtml"
url_resp = requests.get(url=url, headers=self.headers).content.decode('utf-8')
# urls = re.findall(r'<span><a href="(.*?)">.*?</a></span>',url_resp)[:8]
# for i in urls:
# url = "http://www.weather.com.cn" + i
# self.req_pages(url)
elem = etree.HTML(url_resp)
urls = elem.xpath("//ul[@class='lq_contentboxTab2']/li/span/a/@href")
for i in urls:
url = "http://www.weather.com.cn" + i
self.req_pages(url)
# 发送请求,返回数据
def req_pages(self, url):
resp = requests.get(url=url, headers=self.headers).content.decode('utf-8')
elem = etree.HTML(resp)
conMidtab = elem.xpath('//div[@class="conMidtab"]')[0]
if url == "http://www.weather.com.cn/textFC/gat.shtml":
self.parse_pages_gat(resp)
else:
self.parse_pages(conMidtab)
pass
# 解析gat,使用正则表达式
def parse_pages_gat(self, resp):
pattern = re.compile(r'<td width="83" height="23">\s<a href=".*?" target="_blank">(.*?)</a></td>')
city_name = pattern.findall(resp)[:5]
pattern2 = re.compile(r'<td width="86">(.*?)</td>')
min_temp_list = pattern2.findall(resp)[:8]
min_temp = re.findall(r'\d+', ",".join(min_temp_list))
for city, min in zip(city_name, min_temp):
self.content_list.append({"城市": city, "最低气温": min})
# 解析内容
def parse_pages(self, conMidtab):
conMidtab2 = conMidtab.xpath('./div[@class="conMidtab2"]')
for trs in conMidtab2:
tr_list = trs.xpath('.//table//tr')[2:]
for index, tr in enumerate(tr_list):
if index == 0:
city_name = tr.xpath('./td[2]/a/text()')[0]
min_temp = tr.xpath('./td[8]/text()')[0]
self.content_list.append({"城市": city_name, "最低气温": min_temp})
else:
city_name = tr.xpath('./td[1]/a/text()')[0]
min_temp = tr.xpath('./td[7]/text()')[0]
self.content_list.append({"城市": city_name, "最低气温": min_temp})
# 保存数据
def save_data(self):
# 保存到excel
data = pd.DataFrame(self.content_list)
data.to_excel("weather.xlsx", index=False)
# 生成柱形图(前10个最低气温)
self.content_list.sort(key=lambda x: int(x["最低气温"]))
show_data = self.content_list[:10]
bar = Bar()
bar.add_xaxis(list(map(lambda x: x["城市"], show_data)))
bar.add_yaxis('----最低气温排行----', list(map(lambda x: x["最低气温"], show_data)))
bar.render("weather.html")
def main(self):
self.get_url()
# print(self.content_list)
self.save_data()
if __name__ == '__main__':
s = Spider()
s.main()