问题:
想利用python技术爬取网上股票的行情信息
目标网站:
东方财富网 http://quote.eastmoney.com/stocklist.html
百度金融 https://gupiao.baidu.com/stock/
思路:
观察百度金融网站,在它的链接后加上股票id.html就可以获得该股票的行情信息页面
观察东方财富网的源代码可知股票id信息镶嵌在HTML代码中
1.利用requests 访问东方财富网爬取a标签的‘href’属性
2.利用正则表达式获取股票id列表
3.利用得到的列表拼接url链接,一一访问和爬取,得到股票行情信息,并存储到txt文件中
代码:
import requests
from bs4 import BeautifulSoup
import bs4
import re
import traceback
def getHtml(url):
try:
r = requests.get(url)
r.raise_for_status()
r.encoding = 'utf-8' #r.apparent_encoding手动确定编码
return r.text
except:
return ""
def getStocksList(stocksList,html):
soup = BeautifulSoup(html,"html.parser")
a = soup('a')
for i in a:
try:
href = i.attrs['href']
stocksList.append(re.findall(r'[s][hz]\d{6}',href)[0])
except:
continue
def getStockInfo(stocksList,stockInfoUrl,fpath):
for stock in stocksList:
url = stockInfoUrl + stock + '.html'
html = getHtml(url)
try:
if html == "":
continue
infoDict = {}
soup = BeautifulSoup(html,'html.parser')
stockInfo = soup.find('div',attrs={'class':'stock-bets'})
name = stockInfo.find_all(attrs={'class':'bets-name'})[0]
infoDict.update({'股票名称':name.text.split()[0]})
infoName = stockInfo('dt')
infoValue = stockInfo('dd')
for i in range(len(infoName)):
key = infoName[i].text
value = infoValue[i].text
infoDict[key] = value
count = 0
with open(fpath,'a',encoding='utf-8') as f:
f.write(str(infoDict)+'\n')
count = count + 1
print("\r当前进度:{:.2f}%".format(count*100/len(stocksList)),end="")
except:
count = count + 1
print("\r当前进度:{:.2f}%".format(count*100/len(stocksList)),end="")
continue
def main():
stocksListUrl = 'http://quote.eastmoney.com/stocklist.html'
stockInfoUrl = 'https://gupiao.baidu.com/stock/'
fpath = 'D://Info.txt'
ulist=[]
html = getHtml(stocksListUrl)
getStocksList(ulist,html)
getStockInfo(ulist,stockInfoUrl,fpath)
main()