python网络爬虫与数据收集数据分析机器学习期末大作业

1.爬虫

import time
import requests
import re
import csv

class WeatherForecast(object):
    def __init__(self):
        self.url = 'https://tianqi.2345.com/Pc/GetHistory?areaInfo%5BareaId%5D={0}&areaInfo%5BareaType%5D=2&date%5Byear%5D={1}&date%5Bmonth%5D={2}'
        self.headers = {
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.24',
            'accept-encoding': 'gzip, deflate, br'
        }

        self.data_list = []

    # 发送请求获取数据，这里如果是请求text和content得到的响应均是二进制数据,所以我们请求json得到响应是文本内容
    def get_content(self, url):
        res = requests.get(url=url, headers=self.headers)
        content = res.json()
        # print(content['data'])
        return content['data']
    #使用re解析来解析日期，最高气温，最低气温，天气，风力风向，空气质量这些字段。

    def parse_data(self, content):
        result = re.compile(r'<td>(?P<date>.*?)</td>.*?<td style="color:#ff5040;">(?P<max>.*?)</td>'
                            r'.*?<td style="color:#3097fd;" >(?P<min>.*?)</td>.*?<td>(?P<weather>.*?)</td>'
                            r'.*?<td>(?P<cloud>.*?)</td>.*?<td><span class="history-aqi wea-aqi.*?>(?P<sky>.*?)