html股票数据代码,股票数据的网站抓取（4.2）代码优化

#coding=utf-8

from selenium import webdriver

import time

import os

import re

import sys

import threading

import Queue

import Tkinter as tk

from selenium.common.exceptions import NoSuchElementException

def myinit():

reload(sys)

sys.setdefaultencoding('utf8')

#获取屏幕分辨率

def getscreeninfo():

#这个没考虑多显示器的问题

info={}

root = tk.Tk()

info['width_px'] = root.winfo_screenwidth()

info['height_px'] = root.winfo_screenheight()

info['width_mm'] = root.winfo_screenmmwidth()

info['height_mm'] = root.winfo_screenmmheight()

# 2.54 cm = in

info['width_in'] = info['width_mm'] / 25.4

info['height_in'] = info['height_mm'] / 25.4

info['width_dpi'] = info['width_px']/info['width_in']

info['height_dpi'] = info['height_px']/info['height_in']

return info

########################

#生成多个浏览器实例

def makebr(number):

brs=[]

for i in range(number):

br=webdriver.Firefox()

brs.append(br)

return brs

#设置所有浏览器实例大小

def setbrsize(brs,x,y):

for i in brs:

i.set_window_size(200,200)

#排列打开的浏览器

def layoutbrs(brs,x=0,y=0,dx=10,dy=10,mode=0,hang=0):

#mode 0,横向, 1，纵向 2，矩形 3，对角线

#hang 0为尽量按正方形排列

screeninfo=getscreeninfo()

if(mode==0):

dy=0

elif(mode==1):

dx=0

if(mode!=2):

for i in brs:

i.set_window_position(x,y)

x=x+dx

y=y+dy

else:

#这个以后有空再弄

pass

#hang=math.ceil(math.sqrt(len(dbrs)))

#关闭所有浏览器

def closebr(brs):

for i in brs:

i.quit()

#######################

#获得指定股票相关数据

def getinfo(mydriver,gourl):

title='//*[@id="name"]'

code='//*[@id="code"]'

hexinshujuxpath="/html/body/div[14]/div[1]/div[4]/div[1]"

restr="：".decode('utf8')

myre=re.compile(restr,re.I|re.M|re.S)

linetext=""

errorinfo=""

mydriver.get(gourl)

try:

gupiaoming=mydriver.find_element_by_xpath(title).text

gupiaocode=mydriver.find_element_by_xpath(code).text

hexinshuju=mydriver.find_element_by_class_name('pad5')

shujuhang=hexinshuju.find_elements_by_tag_name('tr')

for i in range(len(shujuhang)-2):

shujulie=shujuhang[i].find_elements_by_tag_name('td')

tmpshuju=myre.split(shujulie[0].text)

linetext=linetext+"~"+tmpshuju[1]

shuju=myre.split(shujuhang[8].text)

linetext=linetext+"~"+shuju[1]

tmpshuju=myre.split(shujuhang[9].text)

linetext=linetext+"~"+tmpshuju[1]

linetext="%s~%s%s"%(gupiaoming,gupiaocode,linetext)

#print "数据：",linetext

except NoSuchElementException,e:

#print "不是股票"

pass

except Exception:

errorinfo= "非预期错误"+gourl

print errorinfo

finally:

return linetext,errorinfo

#获得所有股票链接

def geturls(br):

#通过link对象获得链接地址的text文本

def getlinkurl(linklist):

my=[]

for x in linklist:

my.append(x.get_attribute('href'))

return my

sz=[]

sh=[]

br.get("http://quote.eastmoney.com/stocklist.html")

shxpath="/html/body/div[9]/div[2]/div/ul[1]"

szxpath="/html/body/div[9]/div[2]/div/ul[2]"

shgupiao=br.find_element_by_xpath(shxpath)

szgupiao=br.find_element_by_xpath(szxpath)

shgupiaolist=shgupiao.find_elements_by_tag_name('a')

szgupiaolist=szgupiao.find_elements_by_tag_name('a')

sh=getlinkurl(shgupiaolist)

sz=getlinkurl(szgupiaolist)

return sh,sz

#多线程执行用的函数

def thread_getinfo(br,jobslink_queue,jieguo_queue,errorinfo_queue):

while True:

try:

#获得队列里的地址

url=jobslink_queue.get(False) #False =Don't wait

except Queue.Empty:

print "完成退出"

#br.quit()

return

#print url

if(url!=None):

(linetext,errorinfo)=getinfo(br,url)

if(linetext!=""):

jieguo_queue.put(linetext)

if(errorinfo!=""):

errorinfo_queue.put(errorinfo)

#######################

#多线程控制函数

def saveinfoabc(info_filename,error_filename,urllist):

jobslink=Queue.Queue(0)

jieguo=Queue.Queue(0)

errorsinfo=Queue.Queue(0)

for x in urllist[200:250]:#为测试方便这里只取了50，如果要全下载，取消[]就好

jobslink.put(x)

#启动线程

for x in range(THREAD_NUM):

t=threading.Thread(target=thread_getinfo,args=(brs[x],jobslink,jieguo,errorsinfo))

t.start()

f=open(info_filename,'w')

e=open(error_filename,'w')

mycount=0

while (threading.activeCount()>1) or (not jobslink.empty()):

while jieguo.qsize()>0 or errorsinfo.qsize()>0:

if(jieguo.qsize()>0):

jieguotxt=jieguo.get()

f.write(jieguotxt+"\n")

if(errorsinfo.qsize()>0):

error=errorsinfo.get()

e.write(error+"\n")

mycount=mycount+1

if(mycount%100)==0:

print "%d: 活动线程：%d,剩余连接数：%d,结果剩余条数：%d,错误剩余条数:%d"%(mycount,threading.activeCount(),jobslink.qsize(),jieguo.qsize(),errorsinfo.qsize())

time.sleep(0.01)

e.close()

f.close()

print "数据下载完成"

######################

myinit()

br=webdriver.Firefox()

print "获得所有链接地址"

(sh,sz)=geturls(br)

br.quit()

#info_filename='shinfo.txt'

#error_filename='sherror.txt'

THREAD_NUM=10

brs=makebr(THREAD_NUM)

setbrsize(brs,200,300)

layoutbrs(brs ,0,0,80,80,0)

saveinfoabc('shinfo.txt','sherror.txt',sh)

saveinfoabc('szinfo.txt','szerror.txt',sz)

closebr(brs)