python多文件共享变量_Python多处理和共享变量

我不是python的专家,但我已经设法写下了一个多处理代码,它在我的PC中使用了我所有的cpus和内核.我的代码加载了一个非常大的数组,大约1.6 GB,我需要在每个进程中更新数组.幸运的是,更新包括在图像中添加一些人造恒星,每个过程都有一组不同的图像位置,可以添加人造恒星.

图像太大,每次调用一个进程时我都无法创建一个新图像.我的解决方案是在共享内存中创建一个变量,我节省了大量内存.由于某种原因,它适用于90％的图像,但有些区域是我的代码在我之前发送到流程的某些位置添加随机数.它与我创建共享变量的方式有关吗？在我的代码执行过程中,这些进程是否相互干扰？

奇怪的是,当使用单个cpu和单核时,图像是100％完美的,并且图像中没有添加随机数.你建议我在多个进程之间共享一个大型数组吗？这是我的代码的相关部分.请在定义变量im_data时读取行.

import warnings

warnings.filterwarnings("ignore")

from mpl_toolkits.mplot3d import Axes3D

from matplotlib import cm

import matplotlib.pyplot as plt

import sys,os

import subprocess

import numpy as np

import time

import cv2 as cv

import pyfits

from pyfits import getheader

import multiprocessing,Queue

import ctypes

class Worker(multiprocessing.Process):

def __init__(self,work_queue,result_queue):

# base class initialization

multiprocessing.Process.__init__(self)

# job management stuff

self.work_queue = work_queue

self.result_queue = result_queue

self.kill_received = False

def run(self):

while not self.kill_received:

# get a task

try:

i_range,psf_file = self.work_queue.get_nowait()

except Queue.Empty:

break

# the actual processing

print "Adding artificial stars - index range=",i_range

radius=16

x_c,y_c=( (psf_size[1]-1)/2,(psf_size[2]-1)/2 )

x,y=np.meshgrid(np.arange(psf_size[1])-x_c,np.arange(psf_size[2])-y_c)

distance = np.sqrt(x**2 + y**2)

for i in range(i_range[0],i_range[1]):

psf_xy=np.zeros(psf_size[1:3],dtype=float)

j=0

for i_order in range(psf_order+1):

j_order=0

while (i_order+j_order < psf_order+1):

psf_xy += psf_data[j,:,:] * ((mock_y[i]-psf_offset[1])/psf_scale[1])**i_order * ((mock_x[i]-psf_offset[0])/psf_scale[0])**j_order

j_order+=1

j+=1

psf_factor=10.**( (30.-mock_mag[i])/2.5)/np.sum(psf_xy)

psf_xy *= psf_factor

npsf_xy=cv.resize(psf_xy,(npsf_size[0],npsf_size[1]),interpolation=cv.INTER_LANCZOS4)

npsf_factor=10.**( (30.-mock_mag[i])/2.5)/np.sum(npsf_xy)

npsf_xy *= npsf_factor

im_rangex=[max(mock_x[i]-npsf_size[1]/2,0),min(mock_x[i]-npsf_size[1]/2+npsf_size[1],im_size[1])]

im_rangey=[max(mock_y[i]-npsf_size[0]/2,min(mock_y[i]-npsf_size[0]/2+npsf_size[0],im_size[0])]

npsf_rangex=[max(-1*(mock_x[i]-npsf_size[1]/2),min(-1*(mock_x[i]-npsf_size[1]/2-im_size[1]),npsf_size[1])]

npsf_rangey=[max(-1*(mock_y[i]-npsf_size[0]/2),min(-1*(mock_y[i]-npsf_size[0]/2-im_size[0]),npsf_size[0])]

im_data[im_rangey[0]:im_rangey[1],im_rangex[0]:im_rangex[1]] = 10.

self.result_queue.put(id)

if __name__ == "__main__":

n_cpu=2

n_core=6

n_processes=n_cpu*n_core*1

input_mock_file=sys.argv[1]

print "Reading file ",im_file[i]

hdu=pyfits.open(im_file[i])

data=hdu[0].data

im_size=data.shape

im_data_base = multiprocessing.Array(ctypes.c_float,im_size[0]*im_size[1])

im_data = np.ctypeslib.as_array(im_data_base.get_obj())

im_data = im_data.reshape(im_size[0],im_size[1])

im_data[:] = data

data=0

assert im_data.base.base is im_data_base.get_obj()

# run

# load up work queue

tic=time.time()

j_step=np.int(np.ceil( mock_n*1./n_processes ))

j_range=range(0,mock_n,j_step)

j_range.append(mock_n)

work_queue = multiprocessing.Queue()

for j in range(np.size(j_range)-1):

if work_queue.full():

print "Oh no! Queue is full after only %d iterations" % j

work_queue.put( (j_range[j:j+2],psf_file[i]) )

# create a queue to pass to workers to store the results

result_queue = multiprocessing.Queue()

# spawn workers

for j in range(n_processes):

worker = Worker(work_queue,result_queue)

worker.start()

# collect the results off the queue

while not work_queue.empty():

result_queue.get()

print "Writing file ",mock_im_file[i]

hdu[0].data=im_data

hdu.writeto(mock_im_file[i])

print "%f s for parallel computation." % (time.time() - tic)