目录:
- 功能及准备
- 程序结构设计
- 代码
-
功能描述
- 目标:获取上交所和深交所A股的当日的交易信息
- 输出:保存到文件中
- 数据源选择
- 选取原则:股票信息静态存储于html网页中,非js,没有robots限制
- 选取心态:不纠结于个别网站,多找几个网站试试;不纠结于百度,试试google等其它搜索引擎
- 数据源
- 凤凰网财经频道
-
程序结构设计
- 从凤凰网财经频道获取股票信息
- 输出到文件
-
程序实现
from bs4 import BeautifulSoup
import pandas as pd
import requests, bs4, time
class spider(object):
def __init__(self):
self.user_agent = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'}
self.path = 'https://app.finance.ifeng.com/list/stock.php?t='
self.sh = 'sa'
self.sz = 'ha'
self.page = '&f=chg_pct&o=desc&p='
self.ls = []
def getHtmlText(self, url):
try:
req = requests.get(url)
req.raise_for_status()
return req.text
except Exception as e:
return "产生异常:{}".format(e)
def parserHtml(self, html):
soup = BeautifulSoup(html)
for tr in soup.find('table').children:
if isinstance(tr, bs4.element.Tag):
ths = tr('th')
if ths:
self.ls.append([i.string for i in ths])
tds = tr('td')
if tds:
self.ls.append([i.string for i in tds])
def printInfo(self):
# with open('股票大全.txt', 'w') as f:
# f.write(str(self.ls))
pd.DataFrame(self.ls).to_csv('股票大全.csv', encoding='gbk')
def main():
sp = spider()
try:
cnt = 0
while cnt <= 10:
# 获取沪A股
html = sp.getHtmlText(sp.path + sp.sh + sp.page + str(cnt))
sp.parserHtml(html)
# 获取深A股
html = sp.getHtmlText(sp.path + sp.sz + sp.page + str(cnt))
sp.parserHtml(html)
cnt += 1
print(cnt)
except Exception as e:
print('循环了{}次,产生异常:{}'.format(cnt, e))
else:
print('Running normal.')
finally:
sp.printInfo()
print('游戏结束.')
if __name__ == '__main__':
main()