试着自己做了下爬虫,从星座屋网站爬取十二星座30天的运势数据。
import requests
from bs4 import BeautifulSoup
import pandas as pd
#获取12星座的网址
urll='http://www.xzw.com'
r=requests.get(url='http://www.xzw.com/fortune/aries/')
soup=BeautifulSoup(r.text,'lxml')
a=soup.find('div',class_="card_xingzuo").find_all('a')
urllst=[]
for each in a:
urllst.append(urll+each.get('href'))
#获取每个星座30天的数据
datalst=[]
for ui in urllst:
ri=requests.get(url=ui)
soupi=BeautifulSoup(ri.text,'lxml')
alst=soupi.find('div',class_="lday").find_all('a')
urllst2=[]
for each in alst:
urllst2.append(urll+each.get('href'))
for u in urllst2:
r=requests.get(url=u)
soup=BeautifulSoup(r.text,'lxml')
dic={}
lilst1=soup.find(