承接上文:《一种基于linux mmap特性的应用层虚拟内存工具的编写》
java文件
package com.media.cameraAlgorithm.virtualMemoryUtil;
import java.io.File;
public class VirtualMemoryUtil {
static {
System.loadLibrary("virtualmemory");
}
/**create a virtual memory space and return the controller pointer**/
public static native long createVirtualMemory(String path, long vMemSize);
/**put data into the virtual memory and get a virtual memory pointer**/
public static native long putByteArray(long vMemAddress, byte data[]);
public static native void copyNativeDataBlockToVMem(long toNativePointer, long fromNativePointer, long size);
public static native void copyVMemDataToByteArray(long nativePointer, byte jvmByteArray[]);
public static native long virtualMemoryMalloc(long vMemAddress, long size);
/**free the allocated space unit**/
public static native void fileMapFree(long pointer);
public static native void destroyVirtualMemory(long vMemAddress);
/**please use the destroyVirtualMemory method firstly, then clear the virtual memory file**/
public static void cleanVirtualMemoryFile(String path) {
File file = new File(path);
if(file.exists()) {
file.delete();
}
}
}
先编写JNI文件作为使用之前编写的工具的桥梁:
#include "stdio.h"
#include "stdlib.h"
#include <string.h>
#include <jni.h>
#include "android/log.h"
#include "mmapVirtualMemoryUtil.c"
static const char *TAG="mmapVirtualMemoryUtil";
#define LOGI(fmt, args...) __android_log_print(ANDROID_LOG_INFO, TAG, fmt, ##args)
#define LOGD(fmt, args...) __android_log_print(ANDROID_LOG_DEBUG, TAG, fmt, ##args)
#define LOGE(fmt, args...) __android_log_print(ANDROID_LOG_ERROR, TAG, fmt, ##args)
char* jstringToChar(JNIEnv* env, jstring jstr) {
char* rtn = NULL;
jclass clsstring = (*env)->FindClass(env, "java/lang/String");
jstring strencode = (*env)->NewStringUTF(env, "GB2312");
jmethodID mid = (*env)->GetMethodID(env, clsstring, "getBytes", "(Ljava/lang/String;)[B");
jbyteArray barr = (jbyteArray) (*env)->CallObjectMethod(env, jstr, mid, strencode);
jsize alen = (*env)->GetArrayLength(env, barr);
jbyte* ba = (*env)->GetByteArrayElements(env, barr, JNI_FALSE);
if (alen > 0) {
rtn = (char*) malloc(alen + 1);
memcpy(rtn, ba, alen);
rtn[alen] = 0;
}
(*env)->ReleaseByteArrayElements(env, barr, ba, 0);
return rtn;
}
JNIEXPORT jlong Java_com_media_cameraAlgorithm_virtualMemoryUtil_VirtualMemoryUtil_createVirtualMemory(JNIEnv *env, jobject obj, jstring path, jlong vMemSize) {
char *cPath = jstringToChar(env, path);
LOGI("VMem size :%ld, path :%s", (long) vMemSize, cPath);
MemoryUnit *m = createVirtualMemory(cPath, vMemSize, 1);
free(cPath);
return (jlong) m;
}
JNIEXPORT jlong Java_com_media_cameraAlgorithm_virtualMemoryUtil_VirtualMemoryUtil_putByteArray(JNIEnv *env, jobject obj, jlong vMemAddress, jbyteArray data) {
MemoryUnit *m = (MemoryUnit*) vMemAddress;
jbyte *dataArray = (*env)->GetByteArrayElements(env, data, JNI_FALSE);
long len = (long) (*env)->GetArrayLength(env, data);
char* fileMapPoint = (char*) fileMapMalloc(m, len);
if (fileMapPoint == NULL) {
return -1;
}
memcpy(fileMapPoint, dataArray, len);
(*env)->ReleaseByteArrayElements(env, data, dataArray, JNI_FALSE);
return (jlong) fileMapPoint;
}
JNIEXPORT jbyte Java_com_media_cameraAlgorithm_virtualMemoryUtil_VirtualMemoryUtil_getByteFromNative(JNIEnv *env, jobject obj, jlong nativePointer) {
return *(jbyte*) nativePointer;
}
JNIEXPORT void Java_com_media_cameraAlgorithm_virtualMemoryUtil_VirtualMemoryUtil_copyNativeDataBlockToVMem(JNIEnv *env, jobject obj, jlong toNativePointer, jlong fromNativePointer, jlong size) {
memcpy((char*) toNativePointer, (char*) fromNativePointer, size);
}
JNIEXPORT void Java_com_media_cameraAlgorithm_virtualMemoryUtil_VirtualMemoryUtil_copyVMemDataToByteArray(JNIEnv *env, jobject obj, jlong nativePointer, jbyteArray jvmByteArray) {
jbyte *array = (*env)->GetByteArrayElements(env, jvmByteArray, JNI_FALSE);
jlong size = (*env)->GetArrayLength(env, jvmByteArray);
memcpy((char*) array, (char*) nativePointer, size);
(*env)->ReleaseByteArrayElements(env, jvmByteArray, array, JNI_FALSE);
}
JNIEXPORT jlong Java_com_media_cameraAlgorithm_virtualMemoryUtil_VirtualMemoryUtil_virtualMemoryMalloc(JNIEnv *env, jobject obj, jlong vMemAddress, long size) {
return (jlong) fileMapMalloc((MemoryUnit*) vMemAddress, size);
}
JNIEXPORT void Java_com_media_cameraAlgorithm_virtualMemoryUtil_VirtualMemoryUtil_fileMapFree(JNIEnv *env, jobject obj, jlong pointer) {
void *p = (void*) pointer;
fileMapFree(p);
}
JNIEXPORT void Java_com_media_cameraAlgorithm_virtualMemoryUtil_VirtualMemoryUtil_destroyVirtualMemory(JNIEnv *env, jobject obj, jlong vMemAddress) {
MemoryUnit *m = (MemoryUnit*) vMemAddress;
destroyVirtualMemory(m);
}
MMAP工具:
/**@author 陈杰柱 for unit/linux,用于映射外存文件
* 作为内存指针,并模拟malloc和free的操作,来模拟内存条,
* 作为虚拟内存使用,可以同时使用多个虚拟内存条**/
#include "stdio.h"
#include "stdlib.h"
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
/* 虚拟内存条
@param fileMapAddress 内存条起始地址
@param size 内存条大小 */
typedef struct memoryUnit
{
void *fileMapAddress;
unsigned long fileHandle;
size_t size;
} MemoryUnit;
MemoryUnit *createVirtualMemory(char *filePath, size_t size, int createFile);
void *fileMapMalloc(MemoryUnit *mem, size_t size);
int fileMapObjectSize(void *p);
int fileMapFree(void *p);
int destroyVirtualMemory(MemoryUnit *m);
/* 创造虚拟内存条
@param filePath 虚拟内存条在外存的地址
@param size 设定内存条大小
@return 返回内存条对象 */
MemoryUnit *createVirtualMemory(char *filePath, size_t size, int createFile)
{
if(createFile > 0)
{
//创建制定大小的虚拟内存文件
if (creat(filePath, 0755) < 0)
{
printf("create file %s failure!\n", filePath);
return NULL;
}
else
{
printf("create file %s successed!\n", filePath);
}
}
unsigned long f = open(filePath, O_RDWR);
if (lseek(f, size - 1, SEEK_SET) < 0)
{
return NULL;
}
else
{
char data[1] = {0};
write(f, &data, 1);
lseek(f, 0, SEEK_SET);
printf("open file %s successed!\n", filePath);
//创建内存映射
//map the file start position as a FAKE memory address.
MemoryUnit *unit = (MemoryUnit *)malloc(sizeof(MemoryUnit));
unit->fileHandle = f;
unit->fileMapAddress = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, f, 0);
unit->size = size;
if (unit->fileMapAddress == MAP_FAILED) {
free(unit);
return NULL;
}
return unit;
}
}
/* 从虚拟内存条中获取指定大小的连续空间
@param mem 虚拟内存条
@param size 需要分配多大空间
@return 返回连续空间地址 */
void* fileMapMalloc(MemoryUnit *mem, size_t size)
{
long i = 0, count = 0;
void *p = NULL;
size += sizeof(long); //留一个long的位置记录该块的容量
//寻找合适的空块
for (i = 0; i < mem->size; i++)
{
if (*(unsigned char *)(mem->fileMapAddress + i) == 0)
{
count++;
if (count == size)
{
p = mem->fileMapAddress + (i + 1 - size); //找到足够大小的块,减去刚刚遍历越过的位置得到起始地址
*(long *)p = size; //记录真正大小
p += sizeof(long); //掩盖块大小记录标记
break;
}
}
else
{
i += *(long *)(mem->fileMapAddress + i) - 1; //每一个块的头部都是占用空间量,所以直接跳过。块与块之间的空位如果足够使用的话会形成新的块
count = 0;
}
}
return p;
}
/* 已分配空间大小统计
@param p 虚拟内存条分配的连续空间地址
@return 占用虚拟内存条的空间 */
int fileMapObjectSize(void *p)
{
long size = *(long *)(p - sizeof(long));
printf("This fileMapObject's size is %ld\n", size);
return size;
}
/* 销毁已分配空间
@param p 虚拟内存条分配的连续空间地址
@return 是否成功 */
int fileMapFree(void *p)
{
memset(p - sizeof(long), 0, fileMapObjectSize(p));
return 1;
}
int destroyVirtualMemory(MemoryUnit *m)
{
munmap(m->fileMapAddress, m->size);
close(m->fileHandle);
free(m);
return 1;
}
然后第三方闭源库原本需要一个多个大yuv数组,之前是直接把java的byte[]数组通过JNI传给它,现在,是先通过:
mInputImgVMem = VirtualMemoryUtil.createVirtualMemory(vMemPath, 1024L * 1024L * 1024L);
long dataAddress = VirtualMemoryUtil.putByteArray(mInputImgVMem, data1);
mInputImg[mCount] = dataAddress;
先把一系列的yuv数组放进虚拟内存中,并得到没一个yuv数组的虚拟内存首地址(用long保存)。
然后在第三方库中,把传入的虚拟内存地址作为指针给它用:
…………………………………………
jlong *yuvImgAddress = env->GetLongArrayElements(inputImg, NULL);
for (int i = 0; i < imageNumer; i++) {
MByte *yuv_buf_ptr = (MByte *) yuvImgAddress[i]; //A nice trick, make the file have a virtual memory pointer, so the physical one can have more free space.
LoadImage(&pImgInfo.InputImages[i],
yuv_buf_ptr,
width, height);
if (evNumber - 1 < i) {
pImgInfo.InputImagesEV[i] = ev[evNumber - 1];
} else {
pImgInfo.InputImagesEV[i] = ev[i];
}
}
……………………………………
这时,第三方库取yuv数据时,实际取到的不是内存数据,而是保存到文件中的数据,就像欺骗它这个数据在内存中一样,从而在yuv数据并不实际占用内存的同时,让这个需要处理多个内存中的YUV数据的库,能像读内存数据一样直接读外存数据,从而大大节约了物理内存,解除了OOM问题。
即使看起来吃掉4GB“内存”也根本不会OOM