python 多线程

#coding=utf-8
import Queue, MySQLdb
import threading
import datetime, csv, time
from getMongoInfo import getMongoInfo

import sys
reload(sys)
sys.setdefaultencoding('utf8')

objid_que = Queue.Queue(0)

person_que = Queue.Queue(0)

start_time = datetime.datetime.now()

class MyThread(threading.Thread):
    def __init__(self, personidlist, objidlist, threadnum):
        self.peron_queue = personidlist
        # self.objid_queue = objidlist
        self.threadnum = threadnum
        threading.Thread.__init__(self)
    def run(self):
        while True:
            if self.peron_queue.qsize() > 0:
                print 'length of queue:', self.peron_queue.qsize()
                self.getobjid(self.peron_queue.get())
#            else:
            if objid_que.qsize() > 0:
                    print "*******************"
                    try:
                       obj = objid_que.get()
                       self.dojob(obj)
                       print "length of objid_que:", objid_que.qsize()
                    except:
                       writeToFile("1020error", obj)
            if self.peron_queue.qsize() == 0 and objid_que.qsize() == 0:
                time.sleep(5)
                if self.peron_queue.qsize() == 0 and objid_que.qsize() == 0:
                    break
 #               else:
#                    break


    def getobjid(self, person_id):
        try:
           sql = "select objid from djuniqueperson where person = '{0}'".format(person_id)
           objid = getDBdata(sql, '192.168.6.69', 'test', 'test', 'resume_update')[0][0]
           print "objid: ",objid
           objid_que.put(str(objid))
        except:
           print "db error"

    def dojob(self, data):
        objid = data
        print "threadnum:", self.threadnum, objid
        try:
           datainfo = getMongoInfo(objid)
        except:
           print "getmongoinfo error."
        for data in datainfo:
            phone = data["resume_info"]["mobilephone"]
            if not phone or phone == None or len(phone) == 0:
                break
            try:
               hope_job_list = data["deliver_info"][0]["job_intension"]["job_cnt"]  # 解析出期望职位列表
            except:
               print "期望职位不存在."
               hope_job_list = [""]
            hope_job = ""
            for job in hope_job_list:
                hope_job += job.encode('utf-8').replace(",", "/").replace("，","/") + "|"
            try:
                work_num = len(data["resume_info"]["work"])          #工作数目
            except:
                work_num = 0
            recent_job = ""
            if work_num > 0:
                recent_job = data["resume_info"]["work"][0]["job"]      #得到最近一份工作名字
            filename = "result.csv"
            name = ""
            email = ""
            try:
               name = data["resume_info"]["name"]
            except:
               print "名字不存在"
            try:
               email = data["resume_info"]["email"]
            except:
               print "邮箱不存在."

            sql_2 = "select person from djuniqueperson where objid = '{0}'".format(objid)
            person_id = getDBdata(sql_2, '192.168.6.69', 'test', 'test', 'resume_update')[0][0]

            content = person_id + "," + objid + "," + name + "," + phone + "," + email + "," + recent_job + "," + hope_job

            writeToFile("datainfolist.txt", content)

            key_word = ["招聘", "人力", "人事", "HR", "HRBP", "HRD", "HRM", "猎头"]
            for word in key_word:
                print "word: ", word
                print "hope_job: ", hope_job
                print "recent_job: ", recent_job
                if word in hope_job or word in recent_job:
            # if "招聘" in hope_job or "人力" in hope_job or "人事" in hope_job or "HR" in hope_job or "HRBP" in hope_job or "HRD" in hope_job or "HRM" in hope_job or "猎头" in hope_job :
                    print '*********************hit.'
                    filename = "result.csv"
                    writetocsv(filename, [name, phone, email, recent_job, hope_job])
                    break
            break

        end_time = datetime.datetime.now()
        print "start_time: {0}, end_time: {1}".format(start_time, end_time)


def getDBdata(sql, host, user, passwd, db):
    conn = MySQLdb.connect(host=host, user=user, passwd=passwd, db=db, charset='utf8')
    cur = conn.cursor()
    cur.execute('SET NAMES UTF8')
    conn.commit()
    oper = cur.execute(sql)
    data = cur.fetchmany(oper)
    cur.close()
    return data


def writetocsv(filename, datalist):
    with open(filename, 'ab+') as wp:
        writer = csv.writer(wp)
        # writer.writerow(['totalnum', 'norepeatnum'])
        writer.writerow(datalist)


def writeToFile(time, person):
    with open(time[:7] + '.txt', 'ab+') as wp:
        wp.write(person)
        wp.write("\n")
        wp.close()

if __name__ == '__main__':

    for i in range(5):
        MyThread(person_que, objid_que, i).start()

    file = open(sys.argv[1])
    num = 1
    for line in file.xreadlines():
        line = line.replace(" ", "").strip()
        person_que.put(line)
        print num
        num += 1