利用python爬虫大量基金股票的净值写入到csv文件中方便做其他计算

最近公司同事有一个需求就是能够一键抓到股票基金的净值，然后自己做运算，然后空闲时间给他写了个脚本，可能有的人会说python有第三方解析html，但是考虑到给别人用的复杂性，所以这里完全用正则表达式解析

源码会放在下面，利用http://xueqiu.com获取股票的净值   利用http://fund.eastmoney.com获取基金的净值

使用方法介绍，

1.是股票，在雪球网查看自己的股票地址 如  http://xueqiu.com/S/SZ000001 则在数组里填写/SZ000001 注意带/

2.是基金，在http://fund.eastmoney.com网站查询自己的基金地址如 http://fund.eastmoney.com/519983.html 则在数组里填写519983，然后执行脚本，会在当前目录创建一个result.csv文件，然后就可以进行其他运算了

codes =['519983','002529','/02318','/SZ000001','/SH510900','/SZ168101']

codes换成你自己的数组，执行脚本

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import urllib2
import string,re
import os
import csv

import codecs

codes =['519983','002529','/02318','/SZ000001','/SH510900','/SZ168101']

filename = 'result.csv'

def getHtml(url):
    req_header = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
    'Accept':'text/html;q=0.9,*/*;q=0.8',
    'Accept-Charset':'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    'Accept-Encoding':'utf-8',
    'Connection':'close',
    'Referer':None #注意如果依然不能抓取的话，这里可以设置抓取网站的host
    }
    req_timeout = 5
    req = urllib2.Request(url,None,req_header)
    resp = urllib2.urlopen(req,None,req_timeout)
    html = resp.read()
    return html


def parse_one(code):
        #基金
        url = 'http://fund.eastmoney.com/' + code + '.html'
#        print url
        html = getHtml(url)
        pattern ='<dl class="dataItem02">.*?</dl>'
        # 将正则表达式编译成Pattern对象
        m = re.search(pattern,html)
        content = m.group(0)   #dateIemm02
        
        pattern2 = '<dd class="dataNums">.*?</dd>'
        content2 = re.search(pattern2,content).group(0)  #包含净值和比例的标签
        
        pattern3 = '<span\sclass="[^">]+">([^<>]+)</span>'
        content3 = re.search(pattern3,content2)
        price =content3.group(1)
        
        pattern4 = '<dt>.*?</dt>' #带时间的dt->p->
        content4 =re.search(pattern4,content).group(0)
        
        pattern5 = '\(</span>.*?<' #(</span>2016-10-27)<
        content5 = re.search(pattern5,content4).group(0)
        
        time = content5[8: len(content5)-2]
        
        pattern6= '<div class="fundDetail-tit">.*?</div>'
        content6 =re.search(pattern6,html).group(0)  #<div class="fundDetail-tit"><div style="float: left">长信量化先锋混合(<span class="ui-num">519983</span>)</div>
        content6 = content6.replace('<div class="fundDetail-tit">','')
        pattern7 = 'left">.*?<span'
        
        content7 =re.search(pattern7,content6).group(0)  # left">长信量化先锋混合(<span
        name = content7[6:len(content7)-5]
        writeResult(name,code,price,time)

def parse_two(code):
    #股票
    url = 'http://xueqiu.com/S' + code
    html = getHtml(url)
#    print url
    pattern ='stockName"><strong.*?class="stockName">.*?</strong>'
    nameAndCode =re.search(pattern,html).group(0) # stockName"><strong title="E Fund Management Co.,ltd." class="stockName">H股ETF(SH:510900)</strong>


    pattern2 = 'class="stockName">.*?<'
    content1 =re.search(pattern2,nameAndCode).group(0) #class="stockName">H股ETF(SH:510900)<


    nameAndCode =  content1[18:len(content1)-2] #H股ETF(SH:510900
    
    name = nameAndCode.split('(')[0]
    code = nameAndCode.split('(')[1]

    pattern3 = '<div id="currentQuote">.*</div>'  #包含时间净值和时间
    content3= re.search(pattern3,html).group(0)


    pattern4 = '<div class="currentInfo">.*?<div class="stockInfo'
    content4 = re.search(pattern4,content3).group()
   
    #<div class="currentInfo"><strong data-current="40.75" class="stockUp">HK$40.75</strong><span class="stockUp"><span>+0.250</span><span class="quote-percentage">  (+0.62%)</span></span></div><span class="update-quote
    pattern5 = 'data-current=.*class="sto'
    content5 = re.search(pattern5,content4).group()
    
    pattern5 = 'data-current.*?class="'
    content5 = re.search(pattern5,content5).group(0) #data-current="40.75" class="stockUp

    pattern5 = 'ent=.*cl'
    content5 = re.search(pattern5,content5).group()
    
    price = content5.split('\"')[1]
    
    
    pattern6 = '<div class="stockInfo">.*?</div>' #<div class="stockInfo"><span id="timeInfo">10-28 15:27:26 (北京时间)  </span></div>
    content6 = re.search(pattern6,content3).group()
    
    pattern6 = '<span.*?</span>'
    content6 = re.search(pattern6,content6).group()  #<span id="timeInfo">10-28 15:29:00 (北京时间)  </span>
    
    pattern6 = 'timeInfo">.*?<'
    content6 = re.search(pattern6,content6).group()  #<span id="timeInfo">10-28 15:29:00 (北京时间)  </span>

    pattern6 = '>.*?\('
    content6 = re.search(pattern6,content6).group()  #timeInfo">10-28 15:30:10 (北京时间)  <
    time = content6[1:len(content6)-1]

    writeResult(name,code,price,time)



def writeResult(name,code,price,time):
    result =  'name:' + name + '  ' + 'code:'+code + '    ' + '单位净值:' + price +  '  ' + 'time:' + time
    item=[name,code,price,time]
    csvfile = open(filename,'ab')
    csvfile.write(codecs.BOM_UTF8)
    writer = csv.writer(csvfile,dialect='excel')
    writer.writerow(item)
    csvfile.close()
    print '写入成功   结果为:  ' + result



item=['股票名称','代码','单位净值','     时间     ']
csvfile = open(filename,'wb')
csvfile.write(codecs.BOM_UTF8)
writer = csv.writer(csvfile)
writer.writerow(item)
csvfile.close()

for code  in codes:
    if re.match('^\/.*?',code):
        print '\n-----------------------------------股票-------------------------------------------'
        parse_two(code)
    else:
        print '\n-----------------------------------基金-------------------------------------------'
        parse_one(code)


print '\n'