多进程、多线程、单进程处理文件-学习记录

最新推荐文章于 2023-03-22 13:06:35 发布

chl183

最新推荐文章于 2023-03-22 13:06:35 发布

阅读量271

点赞数

分类专栏： Python Linux 文章标签： python 开发语言

本文链接：https://blog.csdn.net/chl183/article/details/128238628

版权

Python 同时被 2 个专栏收录

37 篇文章 16 订阅

订阅专栏

Linux

11 篇文章 1 订阅

订阅专栏

单进程

import csv,shutil

import os
import sys

if len(sys.argv) != 2:
    print('check paramter')
    sys.exit()

dirname = sys.argv[-1]
srcdir = './%s'%dirname
desdir = './%s_txt'%dirname

if not os.path.exists('%s'%desdir):
     os.mkdir('%s'%desdir)
files = os.listdir(srcdir)
for filename in files:
    print(filename)
    a=open('%s/%s'%(srcdir,filename),'r')
    reader = csv.reader(a)
    with open('%s/%s.txt'%(desdir,filename.split('.csv')[0]),'w') as f:
        for i in reader:
            for x in i:
                f.write(x)
                f.write('\x01')
            f.write('\n')
        a.close()

多进程

import csv,shutil

from multiprocessing.dummy import Pool as ThreadPool
import os
import sys
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor


if len(sys.argv) != 2:
    print('check paramter')
    sys.exit()

dirname = sys.argv[-1]
srcdir = './%s'%dirname
desdir = './%s_txt'%dirname

if not os.path.exists('%s'%desdir):
     os.mkdir('%s'%desdir)
files = os.listdir(srcdir)

def process(files):
    print(files)
    a=open('%s/%s'%(srcdir,files),'r')
    reader = csv.reader(a)
    with open('%s/%s.txt'%(desdir,files.split('.csv')[0]),'w') as f:
        for i in reader:
            for x in i:
                f.write(x)
                f.write('\x01')
            f.write('\n')
        a.close()


if __name__=="__main__": 
    pool = ProcessPoolExecutor(max_workers=8)
    for i in files:
        future = pool.submit(process, i)
    pool.shutdown()

多线程

import csv,shutil

from multiprocessing.dummy import Pool as ThreadPool
from multiprocessing import Process

import os
import sys

if len(sys.argv) != 2:
    print('check paramter')
    sys.exit()

dirname = sys.argv[-1]
srcdir = './%s'%dirname
desdir = './%s_txt'%dirname

if not os.path.exists('%s'%desdir):
     os.mkdir('%s'%desdir)
files = os.listdir(srcdir)



def process(files):
    print(files)
    a=open('%s/%s'%(srcdir,files),'r')
    reader = csv.reader(a)
    with open('%s/%s.txt'%(desdir,files.split('.csv')[0]),'w') as f:
        for i in reader:
            for x in i:
                f.write(x)
                f.write('\x01')
            f.write('\n')
        a.close()

if __name__=="__main__": 
    pool = ThreadPool() ###线程池
    pool.map(process, files)
    pool.close()
    pool.join()