DGPU上用共享内存

进程间，有时候想共用硬件内存。如下代码，服务端分配了硬件内存，客户端共享逻辑内存得到一些信息，再读取硬件内存。 sharedMemoryCreate是对shm_open的封装。

服务端

#include <cstdio>
#include <cuda_runtime_api.h>
#include <nvbufsurface.h>
#include "string.h"
#include <unistd.h>
#include "helper_multiprocess.h"
#include "stdlib.h"
#include "helper_cuda.h"

static const char shmName[] = "simpleIPCshm";

typedef struct shmStruct_st {
  cudaIpcMemHandle_t memHandle;
  cudaIpcEventHandle_t eventHandle;
  int device;
  int bufLen;
} shmStruct;

int main() {
  cudaIpcMemHandle_t handle;
  void* ptr = NULL, *ptrDev = NULL;
  char* pbuf = NULL;
  sharedMemoryInfo info;
  cudaStream_t stream;
  cudaEvent_t event;
  volatile shmStruct *shm = NULL;
  int bufLen = 10;

  if (sharedMemoryCreate(shmName, sizeof(*shm), &info) != 0) {
    printf("Failed to create shared memory slab\n");
    exit(EXIT_FAILURE);
  }
  shm = (volatile shmStruct *)info.addr;
  memset((void *)shm, 0, sizeof(*shm));

  /* create memory */
  pbuf = new char[bufLen];
  memset(pbuf, 10, bufLen);
  cudaMalloc(&ptr, bufLen);
  if (cudaMemcpy (ptr, pbuf, bufLen, cudaMemcpyHostToDevice) != cudaSuccess) {
    printf("cudaMemcpy failed\n");
  }

  shm->device = 0;
  shm->bufLen = bufLen;
  checkCudaErrors(cudaSetDevice(shm->device));
  cudaError_t err = cudaIpcGetMemHandle((cudaIpcMemHandle_t *)&shm->memHandle, ptr);
  /*checkCudaErrors(cudaEventCreate(&event));
  printf("event:%d\n", event);
  checkCudaErrors(cudaIpcGetEventHandle((cudaIpcEventHandle_t *)&shm->eventHandle, event));
  cudaSetDevice(shm->device);
*/

   while(1){
    sleep(3);
   }

 /* checkCudaErrors(cudaEventSynchronize(event));
  printf("a checkCudaErrors\n");
  checkCudaErrors(cudaEventDestroy(event)); */
  checkCudaErrors(cudaFree(ptr));
  sharedMemoryClose(&info);
  return 0;
}

客户端

#include <cstdio>
#include <cuda_runtime_api.h>
#include <nvbufsurface.h>
#include "string.h"
#include <unistd.h>
#include "helper_multiprocess.h"
#include "stdlib.h"
#include "helper_cuda.h"

static const char shmName[] = "simpleIPCshm";

typedef struct shmStruct_st {
  cudaIpcMemHandle_t memHandle;
  cudaIpcEventHandle_t eventHandle;
  int device;
  int bufLen;
} shmStruct;



int main() {
  cudaIpcMemHandle_t* cuda_shm_handle;
  sharedMemoryInfo info;
  volatile shmStruct *shm = NULL;

  checkCudaErrors(cudaSetDevice(0));

  if (sharedMemoryOpen(shmName, sizeof(shmStruct), &info) != 0) {
    printf("Failed to create shared memory slab\n");
    exit(EXIT_FAILURE);
  }
  printf("sharedMemoryOpen suc\n");
  shm = (volatile shmStruct *)info.addr;
  int bufLen = shm->bufLen;
  printf("bufLen:%d\n", bufLen);
  void *ptr = NULL;
  cudaError_t err = cudaIpcOpenMemHandle(&ptr, *(cudaIpcMemHandle_t *)&shm->memHandle, cudaIpcMemLazyEnablePeerAccess);
  printf("err:%d\n", err);
  char* pbuf = new char[bufLen];
  memset(pbuf, 10, bufLen);
  if (cudaMemcpy (pbuf, ptr, bufLen, cudaMemcpyDeviceToHost ) != cudaSuccess) {
    printf("cudaMemcpy failed\n");
  }
  for(int i = 0; i < bufLen; i++){
    printf("v:%d\n", pbuf[i]);
  }


  return 0;
}

Makefile

CUDA_VER=12.2
ifeq ($(CUDA_VER),)
  $(error "CUDA_VER is not set")
endif

CXX=g++ -std=c++14

APP:= ipc_server

TARGET_DEVICE = $(shell gcc -dumpmachine | cut -f1 -d -)

CUDA_HOME:= /usr/local/cuda-$(CUDA_VER)
DEEPSTREAM_HOME:= /opt/nvidia/deepstream/deepstream

LIB_INSTALL_DIR?=$(DEEPSTREAM_HOME)/lib/
APP_INSTALL_DIR?=$(DEEPSTREAM_HOME)/bin/

ifeq ($(TARGET_DEVICE),aarch64)
  CFLAGS:= -DPLATFORM_TEGRA
endif

SRCS:= $(wildcard *.cpp)

INCS:= $(wildcard *.h)

PKGS:= gstreamer-1.0 gstreamer-video-1.0 x11 json-glib-1.0

OBJS:= $(patsubst %.c,%.o, $(patsubst %.cpp,%.o, $(SRCS)))

CFLAGS+= \
	-I$(CUDA_HOME)/include \
        -I$(DEEPSTREAM_HOME)/sources/includes 
LIBS+= \
	-L$(CUDA_HOME)/lib64 -lcudart -lcuda \
	-lpthread -lm -ldl -Wl,-rpath

CFLAGS+= $(shell pkg-config --cflags $(PKGS))

LIBS+= $(shell pkg-config --libs $(PKGS))


debug: CXXFLAGS += -DDEBUG -ggdb
debug: CCFLAGS += -DDEBUG -ggdb
debug: CFLAGS += -DDEBUG -ggdb
debug: $(APP)

%.o: %.c $(INCS) Makefile
	$(CC) -c -o $@ $(CFLAGS) $<

%.o: %.cpp $(INCS) Makefile
	$(CXX) -c -o $@ $(CFLAGS) $<

$(APP): $(OBJS) Makefile
	$(CXX) -o $(APP) $(OBJS) $(LIBS)

install: $(APP)
	cp -rv $(APP) $(APP_INSTALL_DIR)

clean:
	rm -rf $(OBJS) $(APP)