python 共享内存 mmap

act50

已于 2022-10-08 15:37:31 修改

阅读量1k

点赞数 1

文章标签： python

于 2022-09-07 16:14:14 首次发布

本文链接：https://blog.csdn.net/act50/article/details/126748057

版权

import mmap
import re
import time


class MemStruct:
  

    def __init__(self, max_u_length=20):
        self.single_length = max_u_length * 3
        self.prms = []
        self.span = b"\x00"
        self.exp = re.compile('\x00*$')

    def add_prm(self, prm):
        if isinstance(prm, list):
            self.prms.extend(prm)
        else:
            self.prms.append(prm)

    def _to_byte(self, prm):
        _b = prm.encode(encoding='UTF-8')
        assert len(_b) < self.single_length, "参数长度大于最大限制了"
        remain = self.single_length - len(_b)
        _b = _b + self.span * remain
        return _b

    def _prm_to_cn_txt(self, prm):
        cn_txt = str(prm, 'utf-8', 'ignore')
        cn_txt = self.exp.sub('', cn_txt)
        return cn_txt

    def to_bytes(self):
        out_list = b''
        for i, b in enumerate(self.prms):
            out_list += self._to_byte(b)
        return out_list

    def to_utf8(self, byte_obj):
        '''会去掉末尾的空行'''
        out_lst = []
        lens = len(byte_obj) // self.single_length
        for i in range(lens):
            start = i * self.single_length
            t = byte_obj[start: start + self.single_length]
            cntxt = self._prm_to_cn_txt(t)
            if len(cntxt)>0:
                out_lst.append(cntxt)
        return out_lst


class MemoryShareHelper:
    def __init__(self, file_name="global_share_memory", max_length=600):
        self.FILE_NAME = file_name
        self.exp = re.compile('\x00*$')
        self.MAX_MEMORY_SIZE = max(len(file_name), max_length)
        self._write_memory_tool = mmap.mmap(0, self.MAX_MEMORY_SIZE, self.FILE_NAME, mmap.ACCESS_WRITE)
        self._read_memory_tool = mmap.mmap(0, self.MAX_MEMORY_SIZE, self.FILE_NAME, mmap.ACCESS_READ)
        self.mem_struct = MemStruct()

    def write_txt_to_memory(self, txt):
        '''
        单行写入内存
        Parameters
        ----------
        txt

        Returns
        -------

        '''
        byte = txt.encode(encoding='UTF-8')
        # python写入共享内存
        try:
            self._write_memory_tool.write(byte)
        except Exception as e:
            print("超出最大限制", e)

    def add_text_to_list(self, txt):
        '''
        待写入的字符加入到列表中
        Parameters
        ----------
        txt

        Returns
        -------

        '''
        self.mem_struct.add_prm(txt)

    def list_to_memory(self):
        '''列表写入到内存中'''
        _bytes = self.mem_struct.to_bytes()
        self.write_byte_to_memory(_bytes)

    def write_byte_to_memory(self, byte_obj):
        '''
        字节写入内存中，单行写人
        Parameters
        ----------
        byte_obj

        Returns
        -------

        '''
        try:
            self._write_memory_tool.write(byte_obj)
        except Exception as e:
            print("超出最大限制", e)

    def read_txt_list_from_memory(self):
        """
        读出一个list
        Returns
        -------

        """
        data = self._read_memory_tool.read(self.MAX_MEMORY_SIZE)
        cn_txt = self.mem_struct.to_utf8(data)
        return cn_txt

    def read_cn_txt_from_memory(self):
        '''
        读单独的一行
        Returns
        -------

        '''
        data = self._read_memory_tool.read(self.MAX_MEMORY_SIZE)
        cn_txt = str(data, 'utf-8', 'ignore')
        cn_txt = self.exp.sub('', cn_txt)
        return cn_txt

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._read_memory_tool.close()
        self._write_memory_tool.close()


if __name__ == '__main__':
    ms = MemoryShareHelper()
    x = '测lalala啦啦啦啦adssddf啦大试.jpg'
    ms.add_text_to_list(x)
    ms.add_text_to_list("测试记测试nihao&9087^(^*$(*97录")
    ms.add_text_to_list("玩哈哈")
    ms.add_text_to_list("你干啥，阿斯顿发斯蒂芬")


    ms.list_to_memory()
    c = ms.read_txt_list_from_memory()
    print(c)
    time.sleep(10)  # 程序退出后内存中就没东西了

['测lalala啦啦啦啦adssddf啦大试.jpg', '测试记测试nihao&9087^(^*$(*97录', '玩哈哈', '你干啥，阿斯顿发斯蒂芬']

进程已结束,退出代码0

上面的写法到了ubuntu里面不灵了，可能是mmap版本升级？

改写一下：

import mmap
import os
import re
import sys
import time

from cjc.tools.memory_share_helper import  MemStruct


class MemoryShareUbuntuHelper:
    '''参照 https://blog.csdn.net/zywvvd/article/details/122778874

         特定大小文件   https://www.h5w3.com/z/347253.html   '''

    def __init__(self, file_name="./memory_share.txt", max_length=40960):
        self.FILE_NAME = file_name
        self.MAX_MEMORY_SIZE = max_length
        self.mem_struct = MemStruct()
        span = b"\x00"
        with open(self.FILE_NAME, 'wb') as f:
            num_chars = span * max_length * 3
            f.write(num_chars)
        self.exp = re.compile('\x00*$')
        # 创建内存映射文件句柄
        self.fd = os.open(self.FILE_NAME, os.O_CREAT | os.O_RDWR)
        # self.fd = os.open(self.FILE_NAME, os.O_CREAT | os.O_TRUNC | os.O_RDWR)
        if sys.platform == 'linux':
            # not win32

            self.write_buf = mmap.mmap(self.fd, max_length, mmap.MAP_SHARED, mmap.PROT_WRITE)
            self.read_buf = mmap.mmap(self.fd, max_length, mmap.MAP_SHARED, mmap.PROT_READ)
        else:
            # win32
            self.write_buf = mmap.mmap(self.fd, max_length, access=mmap.ACCESS_WRITE)
            self.read_buf = mmap.mmap(self.fd, max_length, access=mmap.ACCESS_READ)

    def write_txt_to_memory(self, txt):
        '''
        单行写入内存
        Parameters
        ----------
        txt

        Returns
        -------

        '''
        byte = txt.encode(encoding='UTF-8')
        # python写入共享内存
        try:
            self.write_buf.write(byte)
        except Exception as e:
            print("超出最大限制", e)

    def add_text_to_list(self, txt):
        '''
        待写入的字符加入到列表中
        Parameters
        ----------
        txt

        Returns
        -------

        '''
        self.mem_struct.add_prm(txt)

    def list_to_memory(self):
        '''列表写入到内存中'''
        _bytes = self.mem_struct.to_bytes()
        self.write_byte_to_memory(_bytes)
        self.mem_struct.clear()

    def write_byte_to_memory(self, byte_obj):
        '''
        字节写入内存中，单行写人
        Parameters
        ----------
        byte_obj

        Returns
        -------

        '''
        try:
            self.write_buf.write(byte_obj)
        except Exception as e:
            print("超出最大限制", len(byte_obj), e)

    def read_txt_list_from_memory(self):
        """
        读出一个list
        Returns
        -------
        """
        data = self.read_buf.read(self.MAX_MEMORY_SIZE)
        cn_txt = self.mem_struct.to_utf8(data)
        return cn_txt

    def read_cn_txt_from_memory(self):
        '''
        读单独的一行
        Returns
        -------

        '''
        data = self.read_buf.read(self.MAX_MEMORY_SIZE)
        cn_txt = str(data, 'utf-8', 'ignore')
        cn_txt = self.exp.sub('', cn_txt)
        return cn_txt

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.read_buf.close()
        self.write_buf.close()


if __name__ == '__main__':
    ms = MemoryShareUbuntuHelper()
    x = '测lalala啦v啦啦啦adssddf啦大试.jpg'
    ms.add_text_to_list(x)
    ms.add_text_to_list("测试记测试nihao&9087^(^*$(*97录")
    ms.add_text_to_list("玩哈哈")
    ms.add_text_to_list("你干啥，阿斯顿发斯蒂芬")

    ms.list_to_memory()
    c = ms.read_txt_list_from_memory()
    print(c)
    time.sleep(10)  # 程序退出后内存中就没东西了

又可以愉快的共享了

/home/cjc/anaconda3/envs/yolov7/bin/python3.8 /mnt/d/python/yoloX_deepsort/cjc/tools/memory_share_helper_ubuntu.py
['测lalala啦v啦啦啦adssddf啦大试.jpg', '测试记测试nihao&9087^(^*$(*97录', '玩哈哈', '你干啥，阿斯顿发斯蒂芬']