C++通过pybind11调用Python 实现transpose

在某些场合需要在C++实现类似numpy的numpy.transpose(a, axes)功能，但是很多库如NumCpp都没有提供这样的方法，只有二维矩阵的转置，没法进行多维矩阵任意维度的转换。

比较简单的想法就是利用numpy现有的功能，在c++代码里面通过调用python来调用Numpy的transpose。

直接调用Python提供的原生API接口很麻烦，采用了pybind11可以显著简化调用，特别是涉及到传递numpy和list数据上。

直接用numpy的transpose，因为该函数仅仅是改变array的strides，并不影响内存排布，替换的解决方案则是可以使用TensorFlow的transpose函数，可以得到改变内存排布的结果。后面想到了一个直接使用numpy的简单方法：np做transpose之后reshape(-1)变成一维再reshape到结果维度，可以得到期望stride的内存排布。或者类似Pytorch的contiguous使用ascontiguousarray。

代码如下，读者如果运行可能需要适当修改CMakeLists.txt的include和library path，同时export PYTHONPATH包含.py文件的路径。

cpp main.cpp

#include <iostream >

#include < string >

#include <vector >

using namespace std;

#include <pybind 11 /pybind 11.h >

#include <pybind 11 /stl.h >

#include <pybind 11 /numpy.h >

#include <pybind 11 /embed.h > / / everything needed for embedding

namespace py = pybind 11;

bool Transpose(float * pOutData, float * pInData, vector <int > & inDataShape, vector <int > & perm) {

/ / start the interpreter and keep it alive

py ::scoped_interpreter guard{};

/ / construct numpy array

py ::array_t <float > npInputArray(inDataShape, pInData);

py ::module calc = py ::module ::import( "math_test");

auto func = calc.attr( "transpose");

py :: object result;

try {

result = func(npInputArray, perm);

} catch (std :: exception & e) {

cout < < "call python transpose failed:" < < e.what() < < endl;;

return false;

}

py ::array_t <float > outArray = result.cast <py ::array_t <float >>();

/ / copy output data

py ::buffer_info outBuf = outArray.request();

float * optr = (float *)outBuf.ptr;

memcpy(pOutData, optr, outArray. size() * sizeof(float));

/ / remove ddata manually, result in double free

/ / if (!outArray.owndat a()) {

/ / py ::buffer_info buf = outArray.request();

/ / float * ptr = (float *)buf.ptr;

/ / delete [] ptr;

/ / }

return true;

}

int main(int argc, char * argv[]) {

vector <float > inVec = { 0, 1, 2, 3, 4, 5, 6, 7};

vector <int > shape = { 2, 2, 2};

vector <int > perm = { 2, 1, 0 };

vector <float > outVec = inVec;

Transpose(outVec.dat a(), inVec.dat a(), shape, perm);

cout < < "in data:" < < endl;

for (int elem : inVec) {

cout < < elem < < " ";

}

cout < < endl;

cout < < "out data:" < < endl;

for (int elem : outVec) {

cout < < elem < < " ";

}

cout < < endl;

return 0;

}

python math_test.py

def transpose( data, perm):

import numpy as np

result = np.transpose( data, perm)

resultn = result.reshape(- 1).reshape(result.shape)

return resultn

CMakeLists.txt

cmake_minimum_required(VERSION 3.10)

project(cmake_study LANGUAGES CXX)

set(CMAKE_CXX_ STANDARD 11)

# add_definitions(-D_GLIBCXX_ USE_CXX 11_ABI = 0)

add_executable(main

main.cpp

)

target_include_directories(main

PUBLIC

/usr /include /python 3.6m /

/mnt /d /codes /cpp / call_python /pybind 11- 2.6.1 /include

)

target_link_libraries(

main

PUBLIC

/usr /lib /x 86_ 64-linux-gnu /libpython 3.6m.so

)

一些坑

这个工程单独是可以work的，也就是单纯的cpp单向调用python是可行的，但是如果在python调用cpp代码，而这个cpp代码通过pyblind再调用Python其他模块时行不通。例如我可能是python 启动了tensorflow，然后再tf cpp插件里面调用了numpy的功能(tf的cpp插件调用python的包不要再调用tensorflow)。

针对原生python c api有如下解决方案(不需要Py_Initialize(); Py_Finalize();)：

PyGILState_STATE gstate;

gstate = PyGILState_Ensure();

/* Perform Python actions here. */

result = CallSomeFunction();

/* evaluate result or handle exception */

/* Release the thread. No Python API allowed beyond this point. */

PyGILState_Release(gstate);

经过验证上面的方法对于Pybind11也是适用的，也就是对  py::scoped_interpreter guard{};进行一个替换。

测试代码：

class PyGil {

public:

PyGil() {

gstate = PyGILState_Ensure();

}

~ PyGil() {

PyGILState_Release(gstate);

}

private:

PyGILState_STATE gstate;

};

void testPy() {

cout << "test py begin" << endl;

PyGil gil;

// PyGILState_STATE gstate;

// gstate = PyGILState_Ensure();

py:: module calc = py:: module:: import( "tensorflow");

py:: print( "Hello, world!");

// PyGILState_Release(gstate);

cout << "test py end" << endl;

}

pybind11 cpp调用python的其他案例

pybind11调用函数传递字符串的example:(可见直接传递string即可，无需做转换，极大简化了调用过程)

bool TestStr() {

/ / start the interpreter and keep it alive

py ::scoped_interpreter guard{};

string inStr = "hello world";

py :: object result;

try {

py ::module calc = py ::module ::import( "math_test");

auto func = calc.attr( "test_str");

result = func(inStr);

} catch (std :: exception & e) {

cout < < "call python func failed:" < < e.what() < < endl;;

return false;

}

string outStr = result.cast < string >();

cout < < "out str:" < < outStr < < endl;

return true;

}

总结

从cpp单向通过pybind11调用python时获取Lock用py::scoped_interpreter guard{};，而如果在Python调用cpp，在这个cpp反向再次调用Python可以用上面PyGil 的方式通过gstate = PyGILState_Ensure(); PyGILState_Release(gstate);来实现。

List/vector, string直接传参就好，numpy数组传递转成py::array_t。

pybind11调用相对模块使用xx.yy方式。

参考资料

https://www.jianshu.com/p/c912a0a59af9

https://stackoverflow.com/questions/44659924/returning-numpy-arrays-via-pybind11

https://gist.github.com/terasakisatoshi/79d1f656be9023cc649732c5162b3fc4

https://pybind11.readthedocs.io/en/stable/advanced/embedding.html