import zipfile
import tarfile
import gzip
import os
from time import ctime
from multiprocessing import Pool
from multiprocessing import cpu_count
dayZipsPath = r'.'
quarterZipsPath = r'./tmp'
zipPassWord = b'password'
mrFilePath = r'./data'
def unzipDayFile():
for file_name in os.listdir(dayZipsPath):
if os.path.splitext(file_name)[1] == '.zip':
print( file_name)
file_zip = zipfile.ZipFile(file_name, 'r')
file_zip.extractall(path = quarterZipsPath, pwd = zipPassWord)
file_zip.close()
#os.remove(file_name)
def untarDayFile():
for file_name in os.listdir(dayZipsPath):
if '.tar.gz' in file_name:
print( file_name)
file_tar = tarfile.open(file_name)
file_tar.extractall(path = quarterZipsPath)
file_tar.close()
#os.remove(file_name)
def unzip(zipsList):
for file_name in zipsList:
if os.path.splitext(file_name)[1] == '.zip':
zipFileName = quarterZipsPath +'/'+ file_name
file_zip = zipfile.ZipFile(zipFileName, 'r')
file_zip.extractall(path = mrFilePath, pwd = zipPassWord)
file_zip.close()
os.remove(zipFileName)
if __name__ == '__main__':
print('Begin:%s' % ctime())
#获取CPU核个数
cpuNum = cpu_count()
print(cpuNum)
unzipDayFile()
untarDayFile()
#多进程解压,大大缩短处理时间
quarterZipsList = list(os.listdir(quarterZipsPath))
zipFileNum = len(quarterZipsList)
print("total zip files num:%d" % (zipFileNum))
print("begin unzip:%s" % ctime())
p = Pool()
for i in range(cpuNum):
beginPos = int(i*zipFileNum/cpuNum)
endPos = min(int((i+1)*zipFileNum/cpuNum),zipFileNum)
print("proc %d - %d" % (beginPos, endPos))
p.apply_async(unzip,args=(quarterZipsList[beginPos:endPos],))
print("waiting for unzip quarter mr data ...")
p.close()
p.join()
print("end unzip:%s" % ctime())
print( "End:%s" % ctime())