requests-beautifulsoup-re爬取股票行情信息

问题：

想利用python技术爬取网上股票的行情信息

目标网站：

东方财富网 http://quote.eastmoney.com/stocklist.html

百度金融 https://gupiao.baidu.com/stock/

思路：

观察百度金融网站，在它的链接后加上股票id.html就可以获得该股票的行情信息页面

观察东方财富网的源代码可知股票id信息镶嵌在HTML代码中

 1.利用requests 访问东方财富网爬取a标签的‘href’属性

2.利用正则表达式获取股票id列表

3.利用得到的列表拼接url链接，一一访问和爬取，得到股票行情信息，并存储到txt文件中

 代码：

import requests
from bs4 import BeautifulSoup
import bs4 
import re
import traceback

def getHtml(url):
	try:
		r = requests.get(url)
		r.raise_for_status()
		r.encoding = 'utf-8'  #r.apparent_encoding手动确定编码
		return r.text
	except:
		return ""

def getStocksList(stocksList,html):
	soup = BeautifulSoup(html,"html.parser")
	a = soup('a')
	for i in a:
		try:
			href = i.attrs['href']
			stocksList.append(re.findall(r'[s][hz]\d{6}',href)[0])
		except:
			continue

def getStockInfo(stocksList,stockInfoUrl,fpath):
	for stock in stocksList:
		url = stockInfoUrl + stock + '.html'
		html = getHtml(url)
		try:
			if html == "":
				continue
			infoDict = {}
			soup = BeautifulSoup(html,'html.parser')
			stockInfo = soup.find('div',attrs={'class':'stock-bets'})
			name = stockInfo.find_all(attrs={'class':'bets-name'})[0]
			infoDict.update({'股票名称':name.text.split()[0]})
			infoName = stockInfo('dt')
			infoValue = stockInfo('dd')
			for i in range(len(infoName)):
				key = infoName[i].text
				value = infoValue[i].text
				infoDict[key] = value
			
			count = 0
			with open(fpath,'a',encoding='utf-8') as f:
				f.write(str(infoDict)+'\n')
				count = count + 1
				print("\r当前进度:{:.2f}%".format(count*100/len(stocksList)),end="")
		except:
			count = count + 1
			print("\r当前进度:{:.2f}%".format(count*100/len(stocksList)),end="")
			continue

def main():
	stocksListUrl = 'http://quote.eastmoney.com/stocklist.html'
	stockInfoUrl = 'https://gupiao.baidu.com/stock/'
	fpath = 'D://Info.txt'
	ulist=[]
	html = getHtml(stocksListUrl)
	getStocksList(ulist,html)
	getStockInfo(ulist,stockInfoUrl,fpath)
main()