Python3网络爬虫之抓取城市天气预报

import requests
from lxml import etree
import json
import os

# def get_city_list():

city_url = "https://j.i8tq.com/weather2020/search/city.js"
response = requests.get(city_url)
city_js = response.content.decode()
citys = city_js.split("=")[-1]
# print(citys)
city_dic = json.loads(citys)
# print(city_dic)
# print(city_dic.items())
city_list = []
for k,v in city_dic.items():
    # print(k)
    # print(v)
    for k1,v1 in v.items():
        for k2,v2 in v1.items():
            # print(v2)
            city_list.append(v2);
# print(city_list)
# return city_list

# get_city_list()

input_city = input("请输入你要查询的地区天气：")
status = False
for city in city_list:
    if input_city == city["NAMECN"]:
        status = True
        city_urls = "http://www.weather.com.cn/weather/{}.shtml".format(city["AREAID"])
        print("正则搜索你要查询 {} 的天气....".format(input_city))
        # return city_urls
        # print(city_urls)
        # 发起响应
        response = requests.get(city_urls)
        content = response.content.decode()
        # 提取数据列表
        html_str = etree.HTML(content)
        city = html_str.xpath("//div[@class='crumbs fl']/a[2]/text()")[0]
        town = html_str.xpath("//div[@class='crumbs fl']/a[3]/text()")[0]
        # print(city,town)

        li_list = html_str.xpath("//ul[@class='t clearfix']/li")
        weather_list = []
        for li in li_list:
            item = {}
            item["address"] = city + ">" + town
            item["week_date"] = li.xpath("./h1/text()")[0]
            item["max_temp"] = li.xpath("./p[@class='tem']/span/text()")[0]
            item["min_temp"] = li.xpath("./p[@class='tem']/i/text()")[0].split("℃")[0]
            item["wea"] = li.xpath("./p[@class='wea']/text()")[0]
            print(item)
            weather_list.append(item)

        # 保存成json文件
        file_path = "./source"
        if os.path.exists(file_path):
            with open(file_path + "/weather.json", "w", encoding="utf-8") as f:
                json.dump(weather_list, f, ensure_ascii=False, indent=2)
        else:
            os.mkdir(file_path)
            with open(file_path + "/weather.json", "w", encoding="utf-8") as f:
                json.dump(weather_list, f, ensure_ascii=False, indent=2)