7.3【多线程与多进程】线程间进行事件通知

http://table.finance.yahoo.com/table.csv?s=000001.sz
下载多只股票的csv数据，并将其转换为xml文件
实现一个线程，将转换出的xml文件压缩打包。比如转换线程每生产出100个xml文件，就通知打包线程打包成xxx.tgz文件，并删除xml文件。打包完成后，反向通知转换线程继续转换

线程间的事件通知，可以使用标准库中Threading.Event：
1.等待事件一端调用wait，等待事件
2.通知事件一端调用set，通知事件

#
from threading import Event, Thread
def f(e):
	print 'f 0'
	e.wait()
	print 'f 1'
e = Event()
t = Thread(target=f, args=(e,))
t.start()
# f 0
e.set()
# f 1
e.clear() # 取消e.set()，e.wait()重新开始阻塞

# tar.py 	将当前目录下的.xml文件打包成.tgz
import tarfile
import os
def tarXML(tfname): 
	tf = tarfile.open(tfname,'w:gz')
	for fname in os.listdir('.'):
		if fname.endswith('.xml'):
			tf.add(fname)
			os.remove(fname)
	tf.close()
	if not tf.members:
		os.remove(tfname)

tarXML('test.tgz')

# 另终端
# mkdir tmp
# !tar zxfv test.tgz -C tmp

#
import csv
from xml.etree.ElementTree import Element,ElementTree
import requests
from StringIO import StringIO
from xml_pretty import pretty
# 典型生产者与消费者模型
# 使用标准库中Queue.Queue，它是一个线程安全的队列

# from collections import deuqe
# q = deque() 
# Download线程把下载数据放入队列，Convert线程从队列里提取数据

from Queue import Queue
# 经常使用全局变量是一种不良的设计模式，通过构造器传入

from threading import Event, Thread

class DownloadThread(Thread):
	def __init__(self, sid, queue):
		Thread.__init__(self)
		self.sid =sid
		self.url = 'http://table.finance.yahoo.com/table.csv?s=%s.sz'
		self.url %= str(sid).rjust(6, '0')
		self.queue = queue
		
	def download(self,url):
		response = requests.get(url, timeout=3)
		if response.ok:
			return StirngIO(response.content) 
	
	def run(self):
		print 'Download', self.sid
		#1.进行下载
		data = self.download(self.url)
		#2.将数据(sid, data)传递convert线程
		# # 多个线程访问q时，q是不安全的；对这里临界区代码加锁；或使用线程安全的队列
		# q.append(sid, data) # 在消费端pop出来
		self.queue.put((self.sid, data))

calss ConvertThread(Thread):
	def __init__(self, queue, cEvent, tEvent):
		Thread.__init__(self)
		self.queue = queue
		self.cEvent = cEvent
		self.tEvent = tEvent
	def csvToXml(self, scsv, fxml):
		reader = csv.reader(scsv) 
		headers = reader.next()
		headers = map(lambda h: h.replace(' ',''), headers)

		root = Element('Data') 
		for row in reader:
			eRow = Element('Row')
			root.append(eRow)
			for tag, text in zip(headers, row):
				e = Element(tag)
				e.text = text
				eRow.append(e)
		pretty(root)
		et = ElementTree(root)
		et.write(fxml)
	def run(self):
		count = 0
		# 接收传递过来的数据
		while True: # 只有1个消费者线程，需要循环处理多个生成者生成的数据
			sid, data = self.queue.get()
			print 'Convert', sid
			if sid == -1:
				self.cEvent.set()
				self.tEvent.wait()
				break
			if data:
				fname = str(sid).rjust(6, '0') +'.xml'
				with open(fname, 'wb') as wf:
					self.csvToXml(data, wf)
				count += 1
				if count == 100:
					self.cEvent.set()
					self.tEvent.wait()
					self.tEvent.clear()
					count = 0

import tarfile
import os
class TarThread(Thread):
	def __init__(self, cEvent, tEvent):
		Thread.__init__(self)
		self.count = 0
		self.cEvent = cEvent
		self.tEvent = tEvent
		self.setDaemon(True) # 守护线程
# :70,79s/^/    /
	def tarXML(self): 
		self.count += 1
		tfname = '%d.tgz'% self.count
		tf = tarfile.open(tfname,'w:gz')
		for fname in os.listdir('.'):
			if fname.endswith('.xml'):
				tf.add(fname)
				os.remove(fname)
		tf.close()
		if not tf.members:
			os.remove(tfname)
	def run(self): # 等待转换完成信号，并调用tarXML
		while True:
			self.cEvent.wait()
			self.tarXML()
			self.cEvent.clear()
			self.tEvent.set()

tarXML('test.tgz')

if __name__ == '__main__':
	q = Queue()
	dThreads = [DownloadThread(i, q) for i in xrange(1, 11)]
	cEvent = Event()
	tEvent = Event()
	
	cThread = ConvertThread(q, cEvent,tEvent)
	tThread = TarThread(cEvent,tEvent)
	tThread.start()
	for t in dThreads:
		t.start()
	cThread.start()
	for t in dThreads:
		t.join()
	q.put((-1,None))