获取大 A 所有股票数据

学习练习爬虫实例，获取股票数据

东财威武，没有爬虫限制
https://www.eastmoney.com/robots.txt
获取4千只股票数据保存为 csv

import requests
import re
import json
import time
def getApiRequest(url):
  try:
    headerParams = { 'user-agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36' }
    r= requests.get(url, timeout=30, headers = headerParams)
    r.raise_for_status()
    return r
  except:
    return 'http request error !'

def rinseData(res):
  try:
    jsdata = re.findall(r'\(.*?\)', res.text)
    dicts = json.loads(jsdata[0][1:len(jsdata[0])-1 ])
    diff = dicts.get('data', []).get('diff', [])
    return diff
  except:
    return []

def writeCsv(f, lslist):
  try :
    
    for item in lslist:
      tmp = [item.get('f12'),item.get('f14'),item.get('f15'),item.get('f3'),item['f4'],item['f5'],item['f6'],item['f7'],item['f2'],item['f16'],item['f17'],item['f18'],item['f10'],item['f8'],item['f9'],item['f23'] ]
      datastr = ''
      for s in tmp:
        datastr += str(s) + ','
      f.write(datastr[:len(datastr)-1] + "\n")
  except:
    return 'write csv file Error'

def main():
  total = 236
  page = 20
  start = time.perf_counter() #开始时间
  f = open('gp.csv', 'w+', encoding='utf-8')
  f.write('代码,名称,最新价,涨跌幅,涨跌额,成交量(手),成交额,振幅,最高,最低,今天,昨收,量比,换手率,市盈率(动态),市净率\n')
  for p in range(total):
    a = "*" * p
    b = "." * (total -p)
    c = (p/total)*100
    dur = time.perf_counter() - start
    print("\r{:^3.0f}%[{}->{}]{:.2f}s".format(c,a,b,dur),end="")
    url = 'http://81.push2.eastmoney.com/api/qt/clist/get?cb=jQuery1124017220261478010612_1636785244249&pn='+ str(p) +'&pz='+str(page)+'&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&fid=f3&fs=m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_=1636785244294'
    res = getApiRequest(url)
    lslist = rinseData(res)
    writeCsv(f, lslist)
  f.close()
  print("\n"+"执行结束".center(total//2,"-"))
main()