python invoking c++

---

invoking c/c++ module from python

---

boost python

code

• model_wrapper.cpp

#include <boost/python.hpp>
#include <boost/python/scope.hpp>
#include <boost/python/suite/indexing/vector_indexing_suite.hpp>
#include <boost/python/overloads.hpp>
#include <vector>
#include "module_py.h"

using namespace boost::python;

BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS(write_overloads, MyModulePy::write, 2, 3);
BOOST_PYTHON_MODULE(module_py)
{
    boost::python::scope().attr("__version__") = "v20180123.1";
    class_<MyModulePy>("MyModulePy")
        .def("read", &MyModulePy::read)
        .def("batch_read", &MyModulePy::batch_read)
        .def("write_read", &MyModulePy::batch_write)
        .def("write", &MyModulePy::write, 
			          write_overloads(args("key", "value", "expire"), 
			          "put key-value with a expire time"))
        .def("remove", &MyModulePy::remove)
        .def("config", &MyModulePy::config);
}

• module_py.h

#include "c_module.h" // header of libmodule.so 
struct MyModulePy
{
    MyModulePy()
    {
    }
    std::string read(const std::string& key)
    {
        // do read 
        return mModule.Read(key);
    }
    boost::python::dict batch_read(const boost::python::list& keys)
    {
        std::vector<std::string> ckeys(boost::python::len(keys));
        for (int i = 0; i < boost::python::len(keys); i++)
        {
            ckeys[i] = boost::python::extract<std::string>(keys[i]);
        }
        std::map<std::string, std::string> cvalues = mModule.BatchRead(ckeys);
        boost::python::dict values;
        for (auto it = cvalues.begin(); it != cvalues.end(); ++it)
        {
            values[it->first] = it->second;
        }
        return values;
    }
    void bach_write(const boost::python::dict& keyValues)
	{
	    auto skeys = keyValues.keys();        // http://www.boost.org/doc/libs/1_34_0/libs/python/doc/v2/dict.html
	    int num = boost::python::len(skeys);
	    std::map<std::string, std::string> datas;
	    for (int i = 0; i < num; i++)   // TODO: how to iterate a boost::python::dict ? 
	    {
	        string k = boost::python::extract<std::string>(skeys[i]);    // NOTE: do not use 'const string& k = '
	        string v = boost::python::extract<std::string>(subKeyValues[skeys[i]]);
	        datas[k] = v;
	    }
	    return mModule.batch_write(datas);
    
	}
    void write(const std::string& key, const std::string& value, const int expire = 0)
    {
        // do write
        mModule.Write(key, value, expire);
    }
    void remove(const std::string& key)
    {
	    // do remove 
	    mModule.Delete(key);
    }
    std::string config()
    {
        return mModule.GetConfigInfo();
    }
    Module mModule; // class provided by libmodule.so 
};

• sample.py

import time 
from module_py import MyModulePy 
m = MyModulePy()

key = 'testkey_%s' % time.time()
value = 'testvalue_%s' % time.time()
try:
	print 'config:', m.config()
    m.write(key, value)
    m.write(key, value, 10)
    print 'write:', key, value
    print 'read:', key, m.read(key)
    print 'delete:', key, m.remove(key)
    
    for i in xrange(0, 3):
	    mytair.write('batch-%s' % i, str(i))
    keys = ['batch-%s' % i for i in xrange(0, 3)]
    print mytair.batch_read(keys)
    for each in keys:
        mytair.remove(each)  
    
    print 'read:', key, m.read(key) # exception 
except Exception, e:
    print e

build

g++  model_wrapper.cpp -o module_py.so -shared -lmodule -lboost_python 

---

pybind11

layout

/home/admin/pybind11_sample/
	|---- py_sample.cpp 
	|---- sample.py 
	|---- build.sh
	|---- include/
		|---- pybind11/  i installed pybind11 at here 
	|---- text_processor/ c++ module  

code

• py_sample.cpp

class TextProcessorWrapper
{
public:
    struct InfoWrapper
    {
        std::string simhash_code;
        std::string ancestor_code;
    };
public:
    TextProcessorWrapper(const std::string& data_dir)
    {
        std::map<std::string, std::string> conf;
        conf["data_dir"] = data_dir;
        if (!mTextDedup.Init(conf))
        {
            throw std::runtime_error("failed to init");
        }
    }
    ~TextProcessorWrapper(){}

    InfoWrapper process(const std::string& text, const std::string& title)
    {
        TextInfo info;
        mTextProcessor.Process(text, title, info);
        InfoWrapper _info;
        _info.simhash_code = info.simhashCode;
        _info.ancestor_code = info.ancestorCode;
        return _info;
    }
protected:
    TextProcessor mTextProcessor; // define in libtext_processor_st.a 
};

PYBIND11_MODULE(py_sample, m) 
{
    m.doc() = "text processor  "; // optional module docstring
    // a  py class, __init__(), process() 
    py::class_<TextProcessorWrapper>(m, "TextProcessor")
    .def(py::init<const std::string &>())
    .def("process", &TextProcessorWrapper::process);

	// py class, read only attributes  
    py::class_<TextProcessorWrapper::InfoWrapper>(m, "TextInfo")
    .def_readonly("simhash_code", &TextProcessorWrapper::InfoWrapper::simhash_code)
    .def_readonly("ancestor_code", &TextProcessorWrapper::InfoWrapper::ancestor_code);
} 

• sample.py

#!/usr/bin/env python
# -*- coding=utf-8 -*-
import os
import py_textdedup

data_dir = './data'
dedup = py_sample.TextProcessor(data_dir)

info1 = dedup.process("成都—成温邛高速—温江到寿安的丁字路口左转—成青快速通道—过金马河第一个十字路口右转直行即到",
                    "成都看银杏了")

print info.simhash_code, info.ancestor_code

build

g++ py_sample.cpp -o  py_sample.so  \
-O3 -Wall -shared -std=c++14 -fPIC \
-I ./include \
-I /usr/local/include/python2.7 \
-ltext_processor_st  \
-Wl,--rpath=\$$ORIGIN/libs -Wl,--rpath=\$$ORIGIN 

ref

• https://pybind11.readthedocs.io/en/stable/index.html
• https://blog.csdn.net/fitzzhang/article/details/78988682
• https://www.jianshu.com/p/819e3e8fbe5e
• https://www.jianshu.com/p/e2a48213e241

---

references

1. http://www.boost.org/doc/libs/1_39_0/libs/python/doc/tutorial/doc/html/index.html
2. http://www.boost.org/doc/libs/1_31_0/libs/python/doc/v2/overloads.html#BOOST_PYTHON_MEMBER_FUNCTION_OVERLOADS-spec
3. http://www.boost.org/doc/libs/1_31_0/libs/python/doc/v2/init.html
4. https://stackoverflow.com/questions/3761391/boostpython-python-list-to-stdvector
5. http://www.boost.org/doc/libs/1_41_0/libs/python/doc/v2/indexing.html
6. http://www.boost.org/doc/libs/1_65_1/libs/python/doc/html/reference/high_level_components/boost_python_scope_hpp.html
7. http://www.boost.org/doc/libs/1_34_0/libs/python/doc/v2/dict.html
8. http://www.boost.org/doc/libs/1_37_0/libs/python/doc/v2/list.html
9. https://wiki.python.org/moin/boost.python/extract