SpringBoot整合RocksDB
GitHub: link. 欢迎star
注意：本篇博客风格（不多比比就是撸代码!!!）
一、maven依赖
<!-- https://mvnrepository.com/artifact/org.rocksdb/rocksdbjni -->
<dependency>
<groupId>org.rocksdb</groupId>
<artifactId>rocksdbjni</artifactId>
<version>7.0.3</version>
</dependency>
二、RocksDBUtil.java
import lombok.extern.slf4j.Slf4j;
import org.rocksdb.*;
import org.springframework.util.ObjectUtils;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
/**
* @author Andon
* 2021/12/2
* <p>
* 操作 RocksDB
*/
@Slf4j
public class RocksDBUtil {
private static RocksDB rocksDB;
public static ConcurrentMap<String, ColumnFamilyHandle> columnFamilyHandleMap = new ConcurrentHashMap<>(); //数据库列族(表)集合
public static int GET_KEYS_BATCH_SIZE = 100000;
/*
初始化 RocksDB
*/
static {
try {
String osName = System.getProperty("os.name");
log.info("osName:{}", osName);
String rocksDBPath; //RocksDB文件目录
if (osName.toLowerCase().contains("windows")) {
rocksDBPath = "D:\\RocksDB"; // 指定windows系统下RocksDB文件目录
} else {
rocksDBPath = "/usr/local/rocksdb"; // 指定linux系统下RocksDB文件目录
}
RocksDB.loadLibrary();
Options options = new Options();
options.setCreateIfMissing(true); //如果数据库不存在则创建
List<byte[]> cfArr = RocksDB.listColumnFamilies(options, rocksDBPath); // 初始化所有已存在列族
List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(); //ColumnFamilyDescriptor集合
if (!ObjectUtils.isEmpty(cfArr)) {
for (byte[] cf : cfArr) {
columnFamilyDescriptors.add(new ColumnFamilyDescriptor(cf, new ColumnFamilyOptions()));
}
} else {
columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, new ColumnFamilyOptions()));
}
DBOptions dbOptions = new DBOptions();
dbOptions.setCreateIfMissing(true);
List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(); //ColumnFamilyHandle集合
rocksDB = RocksDB.open(dbOptions, rocksDBPath, columnFamilyDescriptors, columnFamilyHandles);
for (int i = 0; i < columnFamilyDescriptors.size(); i++) {
ColumnFamilyHandle columnFamilyHandle = columnFamilyHandles.get(i);
String cfName = new String(columnFamilyDescriptors.get(i).getName(), StandardCharsets.UTF_8);
columnFamilyHandleMap.put(cfName, columnFamilyHandle);
}
log.info("RocksDB init success!! path:{}", rocksDBPath);
log.info("cfNames:{}", columnFamilyHandleMap.keySet());
} catch (Exception e) {
log.error("RocksDB init failure!! error:{}", e.getMessage());
e.printStackTrace();
}
}
private RocksDBUtil() {
}
/**
* 列族，创建（如果不存在）
*/
public static ColumnFamilyHandle cfAddIfNotExist(String cfName) throws RocksDBException {
ColumnFamilyHandle columnFamilyHandle;
if (!columnFamilyHandleMap.containsKey(cfName)) {
columnFamilyHandle = rocksDB.createColumnFamily(new ColumnFamilyDescriptor(cfName.getBytes(), new ColumnFamilyOptions()));
columnFamilyHandleMap.put(cfName, columnFamilyHandle);
log.info("cfAddIfNotExist success!! cfName:{}", cfName);
} else {
columnFamilyHandle = columnFamilyHandleMap.get(cfName);
}
return columnFamilyHandle;
}
/**
* 列族，删除（如果存在）
*/
public static void cfDeleteIfExist(String cfName) throws RocksDBException {
if (columnFamilyHandleMap.containsKey(cfName)) {
rocksDB.dropColumnFamily(columnFamilyHandleMap.get(cfName));
columnFamilyHandleMap.remove(cfName);
log.info("cfDeleteIfExist success!! cfName:{}", cfName);
} else {
log.warn("cfDeleteIfExist containsKey!! cfName:{}", cfName);
}
}
/**
* 增
*/
public static void put(String cfName, String key, String value) throws RocksDBException {
ColumnFamilyHandle columnFamilyHandle = cfAddIfNotExist(cfName); //获取列族Handle
rocksDB.put(columnFamilyHandle, key.getBytes(), value.getBytes());
}
/**
* 增（批量）
*/
public static void batchPut(String cfName, Map<String, String> map) throws RocksDBException {
ColumnFamilyHandle columnFamilyHandle = cfAddIfNotExist(cfName); //获取列族Handle
WriteOptions writeOptions = new WriteOptions();
WriteBatch writeBatch = new WriteBatch();
for (Map.Entry<String, String> entry : map.entrySet()) {
writeBatch.put(columnFamilyHandle, entry.getKey().getBytes(), entry.getValue().getBytes());
}
rocksDB.write(writeOptions, writeBatch);
}
/**
* 删
*/
public static void delete(String cfName, String key) throws RocksDBException {
ColumnFamilyHandle columnFamilyHandle = cfAddIfNotExist(cfName); //获取列族Handle
rocksDB.delete(columnFamilyHandle, key.getBytes());
}
/**
* 查
*/
public static String get(String cfName, String key) throws RocksDBException {
String value = null;
ColumnFamilyHandle columnFamilyHandle = cfAddIfNotExist(cfName); //获取列族Handle
byte[] bytes = rocksDB.get(columnFamilyHandle, key.getBytes());
if (!ObjectUtils.isEmpty(bytes)) {
value = new String(bytes, StandardCharsets.UTF_8);
}
return value;
}
/**
* 查（多个键值对）
*/
public static Map<String, String> multiGetAsMap(String cfName, List<String> keys) throws RocksDBException {
Map<String, String> map = new HashMap<>(keys.size());
ColumnFamilyHandle columnFamilyHandle = cfAddIfNotExist(cfName); //获取列族Handle
List<ColumnFamilyHandle> columnFamilyHandles;
List<byte[]> keyBytes = keys.stream().map(String::getBytes).collect(Collectors.toList());
columnFamilyHandles = IntStream.range(0, keys.size()).mapToObj(i -> columnFamilyHandle).collect(Collectors.toList());
List<byte[]> bytes = rocksDB.multiGetAsList(columnFamilyHandles, keyBytes);
for (int i = 0; i < bytes.size(); i++) {
byte[] valueBytes = bytes.get(i);
String value = "";
if (!ObjectUtils.isEmpty(valueBytes)) {
value = new String(valueBytes, StandardCharsets.UTF_8);
}
map.put(keys.get(i), value);
}
return map;
}
/**
* 查（多个值）
*/
public static List<String> multiGetValueAsList(String cfName, List<String> keys) throws RocksDBException {
List<String> values = new ArrayList<>(keys.size());
ColumnFamilyHandle columnFamilyHandle = cfAddIfNotExist(cfName); //获取列族Handle
List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
List<byte[]> keyBytes = keys.stream().map(String::getBytes).collect(Collectors.toList());
for (int i = 0; i < keys.size(); i++) {
columnFamilyHandles.add(columnFamilyHandle);
}
List<byte[]> bytes = rocksDB.multiGetAsList(columnFamilyHandles, keyBytes);
for (byte[] valueBytes : bytes) {
String value = "";
if (!ObjectUtils.isEmpty(valueBytes)) {
value = new String(valueBytes, StandardCharsets.UTF_8);
}
values.add(value);
}
return values;
}
/**
* 查（所有键）
*/
public static List<String> getAllKey(String cfName) throws RocksDBException {
List<String> list = new ArrayList<>();
ColumnFamilyHandle columnFamilyHandle = cfAddIfNotExist(cfName); //获取列族Handle
try (RocksIterator rocksIterator = rocksDB.newIterator(columnFamilyHandle)) {
for (rocksIterator.seekToFirst(); rocksIterator.isValid(); rocksIterator.next()) {
list.add(new String(rocksIterator.key(), StandardCharsets.UTF_8));
}
}
return list;
}
/**
* 分片查（键）
*/
public static List<String> getKeysFrom(String cfName, String lastKey) throws RocksDBException {
List<String> list = new ArrayList<>(GET_KEYS_BATCH_SIZE);
// 获取列族Handle
ColumnFamilyHandle columnFamilyHandle = cfAddIfNotExist(cfName);
try (RocksIterator rocksIterator = rocksDB.newIterator(columnFamilyHandle)) {
if (lastKey != null) {
rocksIterator.seek(lastKey.getBytes(StandardCharsets.UTF_8));
rocksIterator.next();
} else {
rocksIterator.seekToFirst();
}
// 一批次最多 GET_KEYS_BATCH_SIZE 个 key
while (rocksIterator.isValid() && list.size() < GET_KEYS_BATCH_SIZE) {
list.add(new String(rocksIterator.key(), StandardCharsets.UTF_8));
rocksIterator.next();
}
}
return list;
}
/**
* 查（所有键值）
*/
public static Map<String, String> getAll(String cfName) throws RocksDBException {
Map<String, String> map = new HashMap<>();
ColumnFamilyHandle columnFamilyHandle = cfAddIfNotExist(cfName); //获取列族Handle
try (RocksIterator rocksIterator = rocksDB.newIterator(columnFamilyHandle)) {
for (rocksIterator.seekToFirst(); rocksIterator.isValid(); rocksIterator.next()) {
map.put(new String(rocksIterator.key(), StandardCharsets.UTF_8), new String(rocksIterator.value(), StandardCharsets.UTF_8));
}
}
return map;
}
/**
* 查总条数
*/
public static int getCount(String cfName) throws RocksDBException {
int count = 0;
ColumnFamilyHandle columnFamilyHandle = cfAddIfNotExist(cfName); //获取列族Handle
try (RocksIterator rocksIterator = rocksDB.newIterator(columnFamilyHandle)) {
for (rocksIterator.seekToFirst(); rocksIterator.isValid(); rocksIterator.next()) {
count++;
}
}
return count;
}
}
三、RocksDBController.java（测试）
import com.andon.springbootutil.domain.ResponseStandard;
import com.andon.springbootutil.dto.RocksDBVo;
import com.andon.springbootutil.util.RocksDBUtil;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import org.rocksdb.RocksDBException;
import org.springframework.web.bind.annotation.*;
import java.util.*;
/**
* @author Andon
* 2021/12/3
*/
@SuppressWarnings("DuplicatedCode")
@Api(tags = "RocksDB")
@RestController
@RequestMapping(value = "/rocksdb")
public class RocksDBController {
@ApiOperation("列族，创建（如果不存在）")
@ApiImplicitParams({
@ApiImplicitParam(name = "cfName", value = "列族", required = true),
})
@PostMapping("/cf")
public ResponseStandard<String> cfAdd(String cfName) throws RocksDBException {
RocksDBUtil.cfAddIfNotExist(cfName);
return ResponseStandard.successResponse(cfName);
}
@ApiOperation("列族，删除（如果存在）")
@ApiImplicitParams({
@ApiImplicitParam(name = "cfName", value = "列族", required = true),
})
@DeleteMapping("/cf")
public ResponseStandard<String> cfDelete(String cfName) throws RocksDBException {
RocksDBUtil.cfDeleteIfExist(cfName);
return ResponseStandard.successResponse(cfName);
}
@ApiOperation("列族名（查询所有）")
@GetMapping("/cf-all")
public ResponseStandard<Set<String>> cfAll() {
Set<String> cfNames = RocksDBUtil.columnFamilyHandleMap.keySet();
ResponseStandard<Set<String>> response = ResponseStandard.successResponse(cfNames);
response.setTotal(cfNames.size());
return response;
}
@ApiOperation("增")
@PostMapping("/put")
public ResponseStandard<RocksDBVo> put(@RequestBody RocksDBVo rocksDBVo) throws RocksDBException {
RocksDBUtil.put(rocksDBVo.getCfName(), rocksDBVo.getKey(), rocksDBVo.getValue());
return ResponseStandard.successResponse(rocksDBVo);
}
@ApiOperation("增（批量）")
@PostMapping("/batch-put")
public ResponseStandard<List<RocksDBVo>> batchPut(@RequestBody List<RocksDBVo> rocksDBVos) throws RocksDBException {
Map<String, String> map = new HashMap<>();
for (RocksDBVo rocksDBVo : rocksDBVos) {
map.put(rocksDBVo.getKey(), rocksDBVo.getValue());
}
RocksDBUtil.batchPut(rocksDBVos.get(0).getCfName(), map);
ResponseStandard<List<RocksDBVo>> response = ResponseStandard.successResponse(rocksDBVos);
response.setTotal(rocksDBVos.size());
return response;
}
@ApiOperation("删")
@ApiImplicitParams({
@ApiImplicitParam(name = "cfName", value = "列族", required = true),
@ApiImplicitParam(name = "key", value = "键", required = true),
})
@DeleteMapping("/delete")
public ResponseStandard<RocksDBVo> delete(String cfName, String key) throws RocksDBException {
String value = RocksDBUtil.get(cfName, key);
RocksDBUtil.delete(cfName, key);
RocksDBVo rocksDBVo = RocksDBVo.builder().cfName(cfName).key(key).value(value).build();
return ResponseStandard.successResponse(rocksDBVo);
}
@ApiOperation("查")
@ApiImplicitParams({
@ApiImplicitParam(name = "cfName", value = "列族", required = true),
@ApiImplicitParam(name = "key", value = "键", required = true),
})
@GetMapping("/get")
public ResponseStandard<RocksDBVo> get(String cfName, String key) throws RocksDBException {
String value = RocksDBUtil.get(cfName, key);
RocksDBVo rocksDBVo = RocksDBVo.builder().cfName(cfName).key(key).value(value).build();
return ResponseStandard.successResponse(rocksDBVo);
}
@ApiOperation("查（多个键值对）")
@PostMapping("/multiGetAsList")
public ResponseStandard<List<RocksDBVo>> multiGetAsList(@RequestBody List<RocksDBVo> rocksDBVos) throws RocksDBException {
List<RocksDBVo> list = new ArrayList<>();
String cfName = rocksDBVos.get(0).getCfName();
List<String> keys = new ArrayList<>(rocksDBVos.size());
for (RocksDBVo rocksDBVo : rocksDBVos) {
keys.add(rocksDBVo.getKey());
}
Map<String, String> map = RocksDBUtil.multiGetAsMap(cfName, keys);
for (Map.Entry<String, String> entry : map.entrySet()) {
RocksDBVo rocksDBVo = RocksDBVo.builder().cfName(cfName).key(entry.getKey()).value(entry.getValue()).build();
list.add(rocksDBVo);
}
ResponseStandard<List<RocksDBVo>> response = ResponseStandard.successResponse(list);
response.setTotal(list.size());
return response;
}
@ApiOperation("查所有键值")
@ApiImplicitParams({
@ApiImplicitParam(name = "cfName", value = "列族", required = true),
})
@GetMapping("/get-all")
public ResponseStandard<List<RocksDBVo>> getAll(String cfName) throws RocksDBException {
List<RocksDBVo> rocksDBVos = new ArrayList<>();
Map<String, String> all = RocksDBUtil.getAll(cfName);
for (Map.Entry<String, String> entry : all.entrySet()) {
RocksDBVo rocksDBVo = RocksDBVo.builder().cfName(cfName).key(entry.getKey()).value(entry.getValue()).build();
rocksDBVos.add(rocksDBVo);
}
ResponseStandard<List<RocksDBVo>> response = ResponseStandard.successResponse(rocksDBVos);
response.setTotal(rocksDBVos.size());
return response;
}
@ApiOperation("分片查（键）")
@ApiImplicitParams({
@ApiImplicitParam(name = "cfName", value = "列族", required = true),
})
@GetMapping("/get-keys")
public ResponseStandard<List<String>> getKeysFrom(String cfName) throws RocksDBException {
List<String> data = new ArrayList<>();
List<String> keys;
String lastKey = null;
while (true) {
keys = RocksDBUtil.getKeysFrom(cfName, lastKey);
if (keys.isEmpty()) {
break;
}
lastKey = keys.get(keys.size() - 1);
data.addAll(keys);
keys.clear();
}
ResponseStandard<List<String>> response = ResponseStandard.successResponse(data);
response.setTotal(data.size());
return response;
}
@ApiOperation("查（所有键）")
@ApiImplicitParams({
@ApiImplicitParam(name = "cfName", value = "列族", required = true),
})
@GetMapping("/get-all-key")
public ResponseStandard<List<String>> getAllKey(String cfName) throws RocksDBException {
List<String> allKey = RocksDBUtil.getAllKey(cfName);
ResponseStandard<List<String>> response = ResponseStandard.successResponse(allKey);
response.setTotal(allKey.size());
return response;
}
@ApiOperation("查总条数")
@ApiImplicitParams({
@ApiImplicitParam(name = "cfName", value = "列族", required = true),
})
@GetMapping("/get-count")
public ResponseStandard<Integer> getCount(String cfName) throws RocksDBException {
int count = RocksDBUtil.getCount(cfName);
return ResponseStandard.successResponse(count);
}
}
四、测试结果
五、特别说明
RocksDB 是C++编写的一个嵌入式的本地的一个高性能的 K-V 数据库，无需安装部署。
很多教程是拉取源码编译运行，实则大可不必。rocksdbjni的源码里api最后都是走的native方法，就是把人家开发数据库的源码功能封装了一层，直接调用就好了。
GitHub: link. 欢迎star
扫一扫
769
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
