目标网站:'https://www.tianqi.com'
爬取内容:34个省会近15天的天气数据
获取34个省会
无网页中的详细信息,前往network\Fetch/XHR中寻找json
找到对应的json后,取到它的地址
而后处理json获取城市名和网址
import json
from pprint import pprint
import requests
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
}
resp = requests.get('https://www.tianqi.com/tianqi/mapdata_new.json', headers=headers)
s = json.loads(resp.text)
# pprint(s)
cities = {i['url'].split('/')[-2]: i['city'] for i in s}
print(cities)
获取每个省会近15天的天气情况
以长沙为例获取单个城市的天气情况:
现需获取日期、天气情况、温度
from lxml import etree
import requests
url = f'https://www.tianqi.com/长沙/15/'
resp = requests.get(url, headers=headers)
parser = etree.HTML(resp.text)
elements = parser.xpath('/html/body/div[7]/div[2]/ul[1]/li/a')
datas = []
for i in elements:
data = i.xpath('./div[1]/span[1]')[0].text
weather_condition = i.xpath('./div[3]')[0].text
temperature = f"{i.xpath('./div[4]/span')[0].text}-{i.xpath('./div[4]/span')[1].text}℃"
datas.append((data, weather_condition, temperature))
print(datas)
现整合上述代码并新增写入excel部分
from lxml import etree
from xlwt import Workbook
from datetime import datetime
from time import sleep
import json
from pprint import pprint
import requests
from random import random
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
}
resp = requests.get('https://www.tianqi.com/tianqi/mapdata_new.json', headers=headers)
s = json.loads(resp.text)
# pprint(s)
cities = {i['url'].split('/')[-2]: i['city'] for i in s}
# print(cities)
def get_weather(city):
url = f'https://www.tianqi.com/{city}/15/'
resp = requests.get(url, headers=headers)
parser = etree.HTML(resp.text)
elements = parser.xpath('/html/body/div[7]/div[2]/ul[1]/li/a')
datas = []
for i in elements:
data = i.xpath('./div[1]/span[1]')[0].text
weather_condition = i.xpath('./div[3]')[0].text
temperature = f"{i.xpath('./div[4]/span')[0].text}-{i.xpath('./div[4]/span')[1].text}℃"
datas.append((data, weather_condition, temperature))
print(datas)
return city, datas
def writeExcel(wb, city, datas):
work_sheet = wb.add_sheet(cities[city], cell_overwrite_ok=True)
work_sheet.write_merge(0, 0, 0, 2, cities[city])
work_sheet.write(1, 0, '日期')
work_sheet.write(1, 1, '天气情况')
work_sheet.write(1, 2, '温度')
for i, k in enumerate(datas):
for j, l in enumerate(k):
work_sheet.write(i + 2, j, l)
def main():
wb = Workbook(encoding='utf-8')
for i in cities.keys():
city, datas = get_weather(i)
writeExcel(wb, city, datas)
sleep(random())
print(i)
wb.save('%s.xlsx' % str(datetime.date(datetime.now())))
if __name__ == '__main__':
main()