利用python爬取天气预报_python实现天气爬虫——利用xpath爬取七天天气预报数据...

python实现天气爬虫——利用xpath爬取七天天气预报数据

import pandas as pd

import lxml

import requests

import csv

from lxml import html

weatherUrl='http://www.weather.com.cn/weather/101280101.shtml'

link2='//li[@class="sky skyid lv{} {}"]'

link1='//li[@class="sky skyid lv{}"]'

#获取目标网页

def getSource(url):

response=requests.get(url)

response.encoding='utf-8'

return response.content #返回网页源代码

selector=lxml.html.document_fromstring(getSource(weatherUrl))

Weather=[]

WeatherLink=[]

WeatherList=[]

for i in range(1,4):

weatherLink1=link1.format(i)

weatherLink2=link2.format(i,'on')

WeatherLink.append(weatherLink1)

WeatherLink.append(weatherLink2)

for j in range(len(WeatherLink)):

for s in range(len(selector.xpath(WeatherLink[j]))):

WeatherList.append(selector.xpath(WeatherLink[j])[s])

for each in WeatherList:

WeatherDict={}

date=each.xpath('h1/text()')[0]

weather=each.xpath('p/@title')[0]

temperature=each.xpath('p[@class="tem"]/i/text()')[0]

num=date[:2]

WeatherDict['date']=date

WeatherDict['weather']=weather

WeatherDict['temperature']=temperature

WeatherDict['num']=num

Weather.append(WeatherDict)

#写入csv

def writeData(Weather):

with open('D:/python/test/weather.csv','w',encoding='utf-8-sig',newline='') as f:

writer=csv.DictWriter(f,fieldnames=['date','weather','temperature','num'])

writer.writeheader()

for each in Weather:

writer.writerow(each)

writeData(Weather) #write in csv ducument

print(Weather) #查看结果