python爬取城市天气

目标网站：'https://www.tianqi.com'

爬取内容：34个省会近15天的天气数据

获取34个省会

无网页中的详细信息，前往network\Fetch/XHR中寻找json

找到对应的json后，取到它的地址 

而后处理json获取城市名和网址

import json
from pprint import pprint
import requests
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
}
resp = requests.get('https://www.tianqi.com/tianqi/mapdata_new.json', headers=headers)
s = json.loads(resp.text)
# pprint(s)
cities = {i['url'].split('/')[-2]: i['city'] for i in s}
print(cities)

获取每个省会近15天的天气情况

以长沙为例获取单个城市的天气情况：

现需获取日期、天气情况、温度 

from lxml import etree
import requests

url = f'https://www.tianqi.com/长沙/15/'
resp = requests.get(url, headers=headers)
parser = etree.HTML(resp.text)
elements = parser.xpath('/html/body/div[7]/div[2]/ul[1]/li/a')
datas = []
for i in elements:
    data = i.xpath('./div[1]/span[1]')[0].text
    weather_condition = i.xpath('./div[3]')[0].text
    temperature = f"{i.xpath('./div[4]/span')[0].text}-{i.xpath('./div[4]/span')[1].text}℃"
    datas.append((data, weather_condition, temperature))
print(datas)

现整合上述代码并新增写入excel部分

from lxml import etree
from xlwt import Workbook
from datetime import datetime
from time import sleep
import json
from pprint import pprint
import requests
from random import random

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
}
resp = requests.get('https://www.tianqi.com/tianqi/mapdata_new.json', headers=headers)
s = json.loads(resp.text)
# pprint(s)
cities = {i['url'].split('/')[-2]: i['city'] for i in s}
# print(cities)


def get_weather(city):
    url = f'https://www.tianqi.com/{city}/15/'
    resp = requests.get(url, headers=headers)
    parser = etree.HTML(resp.text)
    elements = parser.xpath('/html/body/div[7]/div[2]/ul[1]/li/a')
    datas = []
    for i in elements:
        data = i.xpath('./div[1]/span[1]')[0].text
        weather_condition = i.xpath('./div[3]')[0].text
        temperature = f"{i.xpath('./div[4]/span')[0].text}-{i.xpath('./div[4]/span')[1].text}℃"
        datas.append((data, weather_condition, temperature))
    print(datas)
    return city, datas


def writeExcel(wb, city, datas):
    work_sheet = wb.add_sheet(cities[city], cell_overwrite_ok=True)
    work_sheet.write_merge(0, 0, 0, 2, cities[city])
    work_sheet.write(1, 0, '日期')
    work_sheet.write(1, 1, '天气情况')
    work_sheet.write(1, 2, '温度')
    for i, k in enumerate(datas):
        for j, l in enumerate(k):
            work_sheet.write(i + 2, j, l)


def main():
    wb = Workbook(encoding='utf-8')
    for i in cities.keys():
        city, datas = get_weather(i)
        writeExcel(wb, city, datas)
        sleep(random())
        print(i)
    wb.save('%s.xlsx' % str(datetime.date(datetime.now())))


if __name__ == '__main__':
    main()