爬虫基础（7）

目录：

1. 功能及准备
2. 程序结构设计
3. 代码

• 功能描述

  1. 目标：获取上交所和深交所A股的当日的交易信息
  2. 输出：保存到文件中
  3. 数据源选择
     • 选取原则：股票信息静态存储于html网页中,非js，没有robots限制
     • 选取心态：不纠结于个别网站，多找几个网站试试；不纠结于百度，试试google等其它搜索引擎
     • 数据源
       • 凤凰网财经频道

• 程序结构设计

  • 从凤凰网财经频道获取股票信息
  • 输出到文件

• 程序实现

from bs4 import BeautifulSoup
import pandas as pd
import requests, bs4, time

class spider(object):

    def __init__(self):
        self.user_agent = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'}
        self.path = 'https://app.finance.ifeng.com/list/stock.php?t='
        self.sh = 'sa'
        self.sz = 'ha'
        self.page = '&f=chg_pct&o=desc&p='
        self.ls = []

    def getHtmlText(self, url):
        try:
            req = requests.get(url)
            req.raise_for_status()
            return req.text
        except Exception as e:
            return "产生异常：{}".format(e)

    def parserHtml(self, html):
        soup = BeautifulSoup(html)
        for tr in soup.find('table').children:
            if isinstance(tr, bs4.element.Tag):
                ths = tr('th')
                if ths:
                    self.ls.append([i.string for i in ths])
                tds = tr('td')
                if tds:
                    self.ls.append([i.string for i in tds])

    def printInfo(self):
        # with open('股票大全.txt', 'w') as f:
        #     f.write(str(self.ls))
        pd.DataFrame(self.ls).to_csv('股票大全.csv', encoding='gbk')

def main():
    sp = spider()
    try:
        cnt = 0
        while cnt <= 10:
            # 获取沪A股
            html = sp.getHtmlText(sp.path + sp.sh + sp.page + str(cnt))
            sp.parserHtml(html)
            # 获取深A股
            html = sp.getHtmlText(sp.path + sp.sz + sp.page + str(cnt))
            sp.parserHtml(html)
            cnt += 1
            print(cnt)
    except Exception as e:
        print('循环了{}次，产生异常：{}'.format(cnt, e))
    else:
        print('Running normal.')
    finally:
        sp.printInfo()
        print('游戏结束.')

if __name__ == '__main__':
    main()