python文件转码_Python转码&解压&多进程

import zipfile

import tarfile

import gzip

import os

from time import ctime

from multiprocessing import Pool

from multiprocessing import cpu_count

dayZipsPath = r'.'

quarterZipsPath = r'./tmp'

zipPassWord = b'password'

mrFilePath = r'./data'

def unzipDayFile():

for file_name in os.listdir(dayZipsPath):

if os.path.splitext(file_name)[1] == '.zip':

print( file_name)

file_zip = zipfile.ZipFile(file_name, 'r')

file_zip.extractall(path = quarterZipsPath, pwd = zipPassWord)

file_zip.close()

#os.remove(file_name)

def untarDayFile():

for file_name in os.listdir(dayZipsPath):

if '.tar.gz' in file_name:

print( file_name)

file_tar = tarfile.open(file_name)

file_tar.extractall(path = quarterZipsPath)

file_tar.close()

#os.remove(file_name)

def unzip(zipsList):

for file_name in zipsList:

if os.path.splitext(file_name)[1] == '.zip':

zipFileName = quarterZipsPath +'/'+ file_name

file_zip = zipfile.ZipFile(zipFileName, 'r')

file_zip.extractall(path = mrFilePath, pwd = zipPassWord)

file_zip.close()

os.remove(zipFileName)

if __name__ == '__main__':

print('Begin:%s' % ctime())

#获取CPU核个数

cpuNum = cpu_count()

print(cpuNum)

unzipDayFile()

untarDayFile()

#多进程解压，大大缩短处理时间

quarterZipsList = list(os.listdir(quarterZipsPath))

zipFileNum = len(quarterZipsList)

print("total zip files num:%d" % (zipFileNum))

print("begin unzip:%s" % ctime())

p = Pool()

for i in range(cpuNum):

beginPos = int(i*zipFileNum/cpuNum)

endPos = min(int((i+1)*zipFileNum/cpuNum),zipFileNum)

print("proc %d - %d" % (beginPos, endPos))

p.apply_async(unzip,args=(quarterZipsList[beginPos:endPos],))

print("waiting for unzip quarter mr data ...")

p.close()

p.join()

print("end unzip:%s" % ctime())

print( "End:%s" % ctime())