python怎么批量下载年报_如何用Python写一个抓取新浪财经网指定企业年报的脚本...

匿名用户

1级

2017-08-02 回答

1.先得到需要的上市公司的股票代码和名字。

2.分析下载链接地址。以康达尔为例，年报地址，下载链接 的页面 ，链接末尾的8个数字前6个是股票代码，后两位01代表上交所上市公司（股票代码60开头）、02代表深交所上市公司。 让后就可以用一个循环来下载所有的数据！

3.把下载下来的xml文件转化成xls文件，代码如下：

1). xml可能的中文编码错误处理

def xml_Error_C(filename):

fp_xml=open(filename)

fp_x=''#中文乱码改正

for i in range(os.path.getsize(filename)):

i+=1

a=fp_xml.read(1)

if a=='&':

fp_xml.seek(-1,1)

if fp_xml.read(6)==' ':

i+=5

continue

else:

fp_xml.seek(-5,1)

fp_x+=a

fp_xml=open(filename,'w+')

fp_xml.write(fp_x)

fp_xml.flush()

fp_xml.close()

2). xml转xls

def Xmltoxls(xmlname,xlsname):

if os.path.getsize(xmlname)<1024:#小于1K，无该项数据

return False

wb=xlwt.Workbook(encoding='utf-8')

ws=wb.add_sheet('Table')

fp_xml=minidom.parse(xmlname)

root=fp_xml.documentElement

Row=root.getElementsByTagName('Row')

Data=root.getElementsByTagName('Data')

col_num=len(Data)/len(Row)

row_num= 0

for row in Row:

Data=row.getElementsByTagName('Data')

for i in range(col_num):

if len(Data[i].childNodes)==0:

ws.write(row_num,i,' ')

continue

ws.write(row_num,i,Data[i].childNodes[0].nodeValue.strip().encode('utf-8'))

row_num+=1

wb.save(xlsname)

return True

上整体代码：

from xml.dom import minidom

import xlwt

import os,shutil

import time,urllib2

def Xmltoxls(xmlname,xlsname):

if os.path.getsize(xmlname)<1024:

return False

wb=xlwt.Workbook(encoding='utf-8')

ws=wb.add_sheet('Table')

fp_xml=minidom.parse(xmlname)

root=fp_xml.documentElement

Row=root.getElementsByTagName('Row')

Data=root.getElementsByTagName('Data')

col_num=len(Data)/len(Row)

row_num= 0

for row in Row:

Data=row.getElementsByTagName('Data')

for i in range(col_num):

if len(Data[i].childNodes)==0:

ws.write(row_num,i,' ')

continue

ws.write(row_num,i,Data[i].childNodes[0].nodeValue.strip().encode('utf-8'))

row_num+=1

wb.save(xlsname)

return True

def xml_Error_C(filename):

fp_xml=open(filename)

fp_x=''#中文乱码改正

for i in range(os.path.getsize(filename)):

i+=1

a=fp_xml.read(1)

if a=='&':

fp_xml.seek(-1,1)

if fp_xml.read(6)==' ':

i+=5

continue

else:

fp_xml.seek(-5,1)

fp_x+=a

fp_xml=open(filename,'w+')

fp_xml.write(fp_x)

fp_xml.flush()

fp_xml.close()

def errorlog(error):

fp_error=open('errorlog.txt','a')

fp_error.write(error+'\n')

fp_error.close

fp_code=open('stockcode..txt')

fp_basic=open('basicdata_url.txt')

temp='z:\\temp.xml'

for line in fp_code:#设置代码起始位置

if line.split()[0]=='601958':

break

for line in fp_code:#遍历所有代码及名称

filepath='basicdata\\'+line.split()[0]+line.split()[1].replace('*','&')#建立文件夹

if not os.path.isdir(filepath):

os.makedirs(filepath)

for url in fp_basic:#抓取所有数据并保存

url_f=url.split()[0]+line.split()[0]+('01' if int(line.split()[0])>599999 else '02')+'&exp=1'

print 'I am handle '+line+' '+url.split()[1]+' '+'data for you'

filename=filepath+'\\'+line.split()[0]+' '+url.split()[1]+'.xls'

while True:#get xml data

try:

u=urllib2.urlopen(url_f)

time.sleep(0.3)

data=u.read()

f=open(temp,'w+')#保存文件

f.write(data)

f.flush()

f.close()

break

except :

print 'Network error,try latter!'

time.sleep(10)

while True:#xml data to xls data

if url.split()[1] in ['News','Notice','Subject']:

shutil.move(temp,filename) # os.rename("oldname","newname")

break

try:

xml_Error_C(temp)

Xmltoxls(temp,filename)

except IOError:

errorlog('No '+filename)

except:

shutil.move(temp,filename)

errorlog('Not Done '+filename)

break

time.sleep(0.2)

time.sleep(7)

fp_basic.seek(0)

print 'All data have been getted.'

fp_code.close()

fp_basic.close()