python全国天气信息爬取

对应的视频讲解地址请点击：python爬虫实例讲解_哔哩哔哩_bilibilipython爬虫教程，数据信息采集爬取https://www.bilibili.com/video/BV13L4y137Q5?p=1 

#!/usr/bin/env python3
# coding:utf-8
# @Time: 2022/1/17

# 获取全国天气信息（城市和对应的最低气温,保存到excel,并生成柱形图）

import requests  # pip install requests
import re
from lxml import etree
import pandas as pd
from pyecharts.charts import Bar


class Spider:
    def __init__(self):
        self.headers = {
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36"
        }
        self.content_list = []

    # 获取网址
    def get_url(self):
        url = "http://www.weather.com.cn/textFC/hb.shtml"
        url_resp = requests.get(url=url, headers=self.headers).content.decode('utf-8')
        # urls = re.findall(r'<span><a href="(.*?)">.*?</a></span>',url_resp)[:8]
        # for i in urls:
        #     url = "http://www.weather.com.cn" + i
        #     self.req_pages(url)
        elem = etree.HTML(url_resp)
        urls = elem.xpath("//ul[@class='lq_contentboxTab2']/li/span/a/@href")
        for i in urls:
            url = "http://www.weather.com.cn" + i
            self.req_pages(url)

    # 发送请求，返回数据
    def req_pages(self, url):
        resp = requests.get(url=url, headers=self.headers).content.decode('utf-8')
        elem = etree.HTML(resp)
        conMidtab = elem.xpath('//div[@class="conMidtab"]')[0]
        if url == "http://www.weather.com.cn/textFC/gat.shtml":
            self.parse_pages_gat(resp)
        else:
            self.parse_pages(conMidtab)
            pass

    # 解析gat，使用正则表达式
    def parse_pages_gat(self, resp):
        pattern = re.compile(r'<td width="83" height="23">\s<a href=".*?" target="_blank">(.*?)</a></td>')
        city_name = pattern.findall(resp)[:5]
        pattern2 = re.compile(r'<td width="86">(.*?)</td>')
        min_temp_list = pattern2.findall(resp)[:8]
        min_temp = re.findall(r'\d+', ",".join(min_temp_list))
        for city, min in zip(city_name, min_temp):
            self.content_list.append({"城市": city, "最低气温": min})

    # 解析内容
    def parse_pages(self, conMidtab):
        conMidtab2 = conMidtab.xpath('./div[@class="conMidtab2"]')
        for trs in conMidtab2:
            tr_list = trs.xpath('.//table//tr')[2:]
            for index, tr in enumerate(tr_list):
                if index == 0:
                    city_name = tr.xpath('./td[2]/a/text()')[0]
                    min_temp = tr.xpath('./td[8]/text()')[0]
                    self.content_list.append({"城市": city_name, "最低气温": min_temp})
                else:
                    city_name = tr.xpath('./td[1]/a/text()')[0]
                    min_temp = tr.xpath('./td[7]/text()')[0]
                    self.content_list.append({"城市": city_name, "最低气温": min_temp})

    # 保存数据
    def save_data(self):
        # 保存到excel
        data = pd.DataFrame(self.content_list)
        data.to_excel("weather.xlsx", index=False)
        # 生成柱形图(前10个最低气温)
        self.content_list.sort(key=lambda x: int(x["最低气温"]))
        show_data = self.content_list[:10]
        bar = Bar()
        bar.add_xaxis(list(map(lambda x: x["城市"], show_data)))
        bar.add_yaxis('----最低气温排行----', list(map(lambda x: x["最低气温"], show_data)))
        bar.render("weather.html")

    def main(self):
        self.get_url()
        # print(self.content_list)
        self.save_data()


if __name__ == '__main__':
    s = Spider()
    s.main()