Matlab与Python的矩阵数据传输[TCP方法]

目标
网上大多数方法是将矩阵先保存为mat格式，再用python的scipy读取，这种方法不太适合进行实时的操作。因此我打算用Socket通信的方法解决实时性和通用性的问题，这里还涉及到Socket的分包，适合用于数据量较大的场合。最后测试时无线传输一个300MB的数据也就花了10多秒。

目标是从matlab向python通过socket发送一个矩阵数据，经过处理后（奇异值分解），返回结果。

简单版

Python代码（Server）

import socket
import numpy as np
import json
from progress.bar import Bar
#为了将numpy数组进行编码，准备一个json解析numpy数据类
class NpEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.integer):
            return int(obj)
        elif isinstance(obj, np.floating):
            return float(obj)
        elif isinstance(obj, np.ndarray):
            return obj.tolist()
        else:
            return super(NpEncoder, self).default(obj)
addr = ('127.0.0.1',1008) #设置服务端ip地址和端口号
buff_size = 65535 #消息的最大长度(适度调整可以增大数据传输速度)
package_num = 23845 #数据包数量(传输2048*4096 float32矩阵所需)
tcpSerSock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
tcpSerSock.bind(addr)
tcpSerSock.listen(1)

while True:
    print('等待连接...')
    tcpCliSock, addr = tcpSerSock.accept()
    print('连接到:', addr)

    while True:
        data = tcpCliSock.recv(int(buff_size))          #接收数据
        if not data:                                    #检测连接是否断开
            break
        decode = []                                     #存放解码后的数据
        with Bar('Processing', max=package_num) as bar: #进度条
            for i in range(package_num):
                print('%.2f'%(i/package_num*100)+'%')	#数据包接收进度
                recv_data = []
                while len(recv_data)==0:
                    data = tcpCliSock.recv(int(buff_size))
                recv_data = np.frombuffer(data,dtype='>f4') #数据解码
                #数据为4字节浮点(float32)，顺序为big-endian(>)。如接收完整float数据，则使用f8格式
                decode = np.append(decode,recv_data)        #数据拼接
                bar.next()
        print('接收到数据')
        height = int(decode[0])								#矩阵的行
        width = int(decode[1])								#矩阵的列
        echo = decode[2:int(height*width+2)]				#矩阵的数值
        mat = echo.reshape((height,width),order='F')		#按顺序重建矩阵,order参数决定了是行优先还是列优先。
        #完成计算任务
        U, S, V = np.linalg.svd(mat, full_matrices=False)   #奇异值分解
		#将计算结果（三个矩阵）打包为字典
        result={'U':U,
                'S':S,
                'V':V}
        send_result = json.dumps(result,cls=NpEncoder)      #将字典编码
        print('需要发回%d字节的数据包'%len(send_result))
        tcpCliSock.send(send_result.encode('utf-8'))        #数据发送
        break
    tcpCliSock.close()
tcpSerSock.close()

 MATLAB代码

close all;clear;clc;
data = randi(10,4096,2048);
config = whos('data');
package_num = 68860; %接收数据包的个数
time_out = 0.5; % 空数据包的等待时间
tcpipClient = tcpip('127.0.0.1',1008,'NetworkRole','Client');%服务器地址
set(tcpipClient,'OutputBufferSize',65535);%数据包的大小
pause(0.5)
set(tcpipClient,'Timeout',time_out);
fopen(tcpipClient);
disp("连接成功")
disp("数据发送")
send_data = [config.size(:);data(:)];
config_send = whos('send_data');
fwrite(tcpipClient,send_data,'float32');
disp("数据接收")
recv_data = [];
h=waitbar(0,'正在接收数据');
for i=1:package_num  %因为一个数据包大小有限，需要重复多次接收
    recv_package = [];
    waitbar(i/package_num)
    while isempty(recv_package)
        recv_package=fread(tcpipClient);
    end
    recv_data = vertcat(recv_data,recv_package);
end
close(h)
str = convertCharsToStrings(native2unicode(recv_data,'utf-8'));%解码出字典
try
    dic = jsondecode(str);%将json形式的字典数据里面的矩阵数据提取
    U=dic.U;
    S=dic.S;
    V=dic.V;
    re = U*diag(S)*V;%验证在python计算的结果是否和matlab算出来的一致
catch
    disp('WARNNING:接收数据包数量(package_num)设置太小')
end
disp('连接断开')
fclose(tcpipClient);

存在问题
这个简单版本的代码可以运行，但是有以下问题:

Python端若接收的是空数据包会一直等待。因此package_num要设置的不大不小刚刚好。
Matlab接收回来的数据包是空数据包时也会等待直至超时，若设置的数据包数量太多传输回来很耗时间。
复杂版
为了解决上述的问题，我进行了以下调整。在进行正式的数据传输前增加一次通信，告知数据大小，推算出数据包长度，不再需要手动设置数据包个数。同时通信的速度也有所提升。

Python代码
上边有的注释这里就不赘诉了。
 

Matlab代码

close all;clear;clc;
data = double(rand(2048,2048));
config = whos('data');
time_out = 60; % 投送数据包的等待时间
tcpipClient = tcpip('127.0.0.1',1008,'NetworkRole','Client');
set(tcpipClient,'OutputBufferSize',67108880+64);%2048*4096
set(tcpipClient,'Timeout',time_out);
tcpipClient.InputBufferSize = 8388608;%8M
tcpipClient.ByteOrder = 'bigEndian';
fopen(tcpipClient);
disp("连接成功")
disp("数据发送")

send_data = [config.size(:);data(:)];
config_send = whos('send_data');
fwrite(tcpipClient,[config_send.bytes/2;send_data],'float32');
disp("数据接收")
recv_data = [];

%重复多次接收
h=waitbar(0,'正在接收数据');
while isempty(recv_data)
    recv_data=fread(tcpipClient);%读取第一组数据
end
header = convertCharsToStrings(native2unicode(recv_data,'utf-8'));
recv_bytes = str2double(regexp(header,'(?<=(L": )).*?(?=(,|$))','match'))-2;%正则化提取数据大小
while length(recv_data)<recv_bytes
    if recv_data(end)==125
        break
    end
    waitbar(length(recv_data)/recv_bytes)
    recv_package = [];
    while isempty(recv_package)
        try
            recv_package=fread(tcpipClient);
        catch
            continue
        end
    end
    recv_data = vertcat(recv_data,recv_package);
end
close(h)
chararray = native2unicode(recv_data,'utf-8');
str = convertCharsToStrings(chararray);
try
    U = dic.U;
    S = dic.S;
    V = dic.V;
    re = U*diag(S)*V;
catch
    disp('WARNNING:接收不完全')
end
disp('连接断开')
fclose(tcpipClient);

       

多个矩阵的传输（例子：复数数据）

有时需要求解复数数据，把复数分解为实部和虚部两个矩阵。

Python代码

   

import socket
import numpy as np
import json
from progress.bar import Bar
import math
#json解析numpy数据类
class NpEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.integer):
            return int(obj)
        elif isinstance(obj, np.floating):
            return float(obj)
        elif isinstance(obj, np.ndarray):
            return obj.tolist()
        else:
            return super(NpEncoder, self).default(obj)
addr = ('127.0.0.1',1008) #设置服务端ip地址和端口号
buff_size = 65535         #消息的最大长度
tcpSerSock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
tcpSerSock.bind(addr)
tcpSerSock.listen(1)
tcpSerSock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 65535)
while True:
    print('等待连接...')
    tcpCliSock, addr = tcpSerSock.accept()
    print('连接到:', addr)

    while True:
        decode = []                                   #存放解码后的数据
        recv_data = []
        while not recv_data:
            recv_data = tcpCliSock.recv(int(buff_size))
        data_bytes = np.frombuffer(bytes(recv_data),count=1,dtype='>f4')
        print('正在接收...')
        while len(recv_data) < data_bytes[0]:
            data = []
            while not data:
                data = tcpCliSock.recv(int(buff_size))
            recv_data += data              #数据拼接
        data_bytes = np.frombuffer(bytes(recv_data),count=1,dtype='>f4')
        print(data_bytes,len(recv_data))
        decode = np.frombuffer(bytes(recv_data),dtype='>f4')     #数据解码                
        print('接收到数据')
        height = int(decode[1])
        width = int(decode[2])
        echo_r = decode[3:int(height*width+3)]
        echo_i = np.array(decode[int(height*width+3):])*1.j
        echo = echo_r + echo_i
        mat = echo.reshape((height,width),order='F')
        U, S, V = np.linalg.svd(mat, full_matrices=False)   #奇异值分解
        #U, S, V = svd(mat)
        result={'L':2e8,
                'U':np.real(U),
                'S':np.real(S),
                'V':np.real(V),
                'Ui':np.imag(U),
                'Si':np.imag(S),
                'Vi':np.imag(V)}
        send_result = json.dumps(result,cls=NpEncoder)      #将字典编码
        result['L'] = len(send_result)
        matlab_buffer = 33554432
        fill_space = math.ceil(result['L']/matlab_buffer+1)*matlab_buffer
        send_result = json.dumps(result,cls=NpEncoder).ljust(fill_space).encode('utf-8')#重编码
        print('需要发回%d字节的数据包'%len(send_result))
        tcpCliSock.sendto(send_result,addr) #数据发送
        print('发送完成')
        break
    tcpCliSock.close()
tcpSerSock.close()

 Matlab代码

close all;clear;clc;
%data = double(rand(2048,2048));
data = load('Echo2048_4096.mat').Echo;
config = whos('data');
time_out = 60; % 投送数据包的等待时间
tcpipClient = tcpip('127.0.0.1',1008,'NetworkRole','Client');
set(tcpipClient,'OutputBufferSize',67108880+64);%2048*4096
set(tcpipClient,'Timeout',time_out);
%tcpipClient.InputBufferSize = 8388608;%8M
tcpipClient.InputBufferSize = 33554432;%32M
tcpipClient.ByteOrder = 'bigEndian';
fopen(tcpipClient);
disp("连接成功")
disp("数据发送")

type = 'complex';
if strcmp(type,'complex')
    rmap = real(data);
    imap = imag(data);
    send_data = [config.size(:);rmap(:);imap(:)];
    config_send = whos('send_data');
    fwrite(tcpipClient,[config_send.bytes/2;send_data],'float32');
else
    send_data = [config.size(:);data(:)];
    config_send = whos('send_data');
    fwrite(tcpipClient,[config_send.bytes/2;send_data],'float32');
end
disp("数据接收")
recv_data = [];

%重复多次接收
h=waitbar(0,'正在接收数据');
while isempty(recv_data)
    recv_data=fread(tcpipClient);%读取第一组数据
end
header = convertCharsToStrings(native2unicode(recv_data,'utf-8'));
recv_bytes = str2double(regexp(header,'(?<=(L": )).*?(?=(,|$))','match'))-2;
while length(recv_data)<recv_bytes
    if recv_data(end)==125
        break
    end
    waitbar(length(recv_data)/recv_bytes)
    recv_package = [];
    while isempty(recv_package)
        try
            recv_package=fread(tcpipClient);
        catch
            continue
        end
    end
    recv_data = vertcat(recv_data,recv_package);
end
close(h)
chararray = native2unicode(recv_data,'utf-8');
str = convertCharsToStrings(chararray);
try
    dic = jsondecode(str);
    if strcmp(type,'complex')
        U = dic.U + dic.Ui*1.j;
        S = dic.S + dic.Si*1.j;
        V = dic.V + dic.Vi*1.j;
    else
        U = dic.U;
        S = dic.S;
        V = dic.V;
    end
    re = U*diag(S)*V;
catch
    disp('WARNNING:接收不完全')
end
disp('连接断开')
fclose(tcpipClient);