public interface ICacheManager {
Cache getOrCreateCache(String paramString);
Cache getOrCreateCache(String paramString, CacheLoader<String, Map<String, Object>> paramCacheLoader);
}
import com.example.manager.ICacheManager;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@Component
public class CacheManager implements ICacheManager {
private static final Logger log = LoggerFactory.getLogger(CacheManager.class);
private Map<String, Cache> cacheMap = new HashMap<>();
@Value("#{${cache.maximumSize.map:null}}")
private Map<String, Long> cacheMaximumSizeMap;
@Value("${cache.default.maximumSize}")
private Long defaultCacheMaximumSize;
@Value("#{${cache.expireAfterAccess.map:null}}")
private Map<String, Long> cacheExpireAfterAccessMap;
@Value("${cache.default.expireAfterAccess}")
private Long defaultCacheExpireAfterAccess;
@Autowired
private Environment env;
public Cache getOrCreateCache(String cacheName) {
Cache cache = this.cacheMap.get(cacheName);
if (cache != null)
return cache;
synchronized (CacheManager.class) {
if (!this.cacheMap.containsKey(cacheName)) {
Long maximumSize = getCacheProperty(cacheName, this.cacheMaximumSizeMap, this.defaultCacheMaximumSize);
Long expireAfterAccess = getCacheProperty(cacheName, this.cacheExpireAfterAccessMap, this.defaultCacheExpireAfterAccess);
cache = CacheBuilder.newBuilder().maximumSize(maximumSize.longValue()).expireAfterAccess(expireAfterAccess.longValue(), TimeUnit.MILLISECONDS).build();
this.cacheMap.put(cacheName, cache);
log.info("create cache, cacheName: {}, maximumSize: {}, expireAfterAccess: {}ms", new Object[] { cacheName, maximumSize, expireAfterAccess });
}
}
return this.cacheMap.get(cacheName);
}
public Cache getOrCreateCache(String cacheName, CacheLoader<String, Map<String, Object>> cacheLoader) {
Cache cache = this.cacheMap.get(cacheName);
if (cache != null)
return cache;
synchronized (CacheManager.class) {
if (!this.cacheMap.containsKey(cacheName)) {
Long maximumSize = getCacheProperty(cacheName, this.cacheMaximumSizeMap, this.defaultCacheMaximumSize);
Long expireAfterAccess = getCacheProperty(cacheName, this.cacheExpireAfterAccessMap, this.defaultCacheExpireAfterAccess);
LoadingCache loadingCache = CacheBuilder.newBuilder().maximumSize(maximumSize.longValue()).refreshAfterWrite(expireAfterAccess.longValue(), TimeUnit.MILLISECONDS).build(cacheLoader);
this.cacheMap.put(cacheName, loadingCache);
log.info("create cache, cacheName: {}, maximumSize: {}, expireAfterAccess: {}ms", new Object[] { cacheName, maximumSize, expireAfterAccess });
}
}
return this.cacheMap.get(cacheName);
}
private <T> T getCacheProperty(String cacheName, Map<String, T> configMap, T defaultValue) {
if (configMap == null || !configMap.containsKey(cacheName))
return defaultValue;
return configMap.get(cacheName);
}
public int getCacheMapSize() {
if (this.cacheMap != null)
return this.cacheMap.size();
return 0;
}
}
@Component
public class MetabaseInfoCache implements Serializable {
private static final Logger log = LoggerFactory.getLogger(MetabaseInfoCache.class);
@Value("${queryResourceFromMetabase:false}")
private Boolean queryResourceFromMetabase;
@Value("${queryApiWeaknessEnable:false}")
private Boolean queryApiWeaknessEnable;
@DynamicValue(dataId = "handler.metabase.mapping.json", groupId = "handler", typeClz = MetabaseMappingConfig.class)
private MetabaseMappingConfig metabaseMappingConfig;
private static Cache<String, Optional<Map<String, Object>>> apiCache;
private static Cache<String, Optional<Map<String, Object>>> appCache;
@Autowired
private MetabaseClientProxy metabaseClientProxy;
@Autowired
private MongoClientProxy mongoClientProxy;
@Autowired
MongoTemplate mongoTemplate;
public static final String LABEL_CONFIGS_KEY = "labelConfigs";
public static final String WEAKNESS_ID_KEY = "weaknessId";
public static final String ENUMERATE_PARA_LABEL_CONFIGS_KEY = "enumerateParaLabelConfigs";
public static final String RECOMMEND_FLG_KEY = "recommendFlag";
public static final String CLASSIFICATIONS_KEY = "classifications";
public static final String FEATURE_LABELS_KEY = "featureLabels";
public static final Map<String, Object> EMPTY_MAP = new HashMap<>();
public MetabaseInfoCache(@Autowired ICacheManager cacheManager) {
apiCache = cacheManager.getOrCreateCache("apiCache", new CacheLoader<String, Map<String, Object>>() {
public Map<String, Object> load(String key) throws Exception {
return MetabaseInfoCache.this.loadApiInfo(key);
}
});
appCache = cacheManager.getOrCreateCache("appCache", new CacheLoader<String, Map<String, Object>>() {
public Map<String, Object> load(String key) throws Exception {
return MetabaseInfoCache.this.loadAppInfo(key);
}
});
}
public Map<String, Object> getApiInfos(String uri) {
try {
return ((Optional<Map<String, Object>>)apiCache.get(uri, () -> Optional.ofNullable(loadApiInfo(uri)))).orElse(EMPTY_MAP);
} catch (ExecutionException e) {
log.warn("get http api resource from cache failed, uri: {}", uri, e);
return EMPTY_MAP;
}
}
public Map<String, Object> getAppInfos(String uri) {
try {
return ((Optional<Map<String, Object>>)appCache.get(uri, () -> Optional.ofNullable(loadAppInfo(uri)))).orElse(EMPTY_MAP);
} catch (ExecutionException e) {
log.warn("get http app resource from cache failed, uri: {}", uri, e);
return EMPTY_MAP;
}
}
private Map<String, Object> loadApiInfo(String apiUri) {
Map<String, Object> apiInfos = new HashMap<>();
Map<String, MetabaseMappingConfig.FiledDesc> mappings = this.metabaseMappingConfig.getHttpApiMappings();
Monitor.TransactionInfo t = Monitor.newTransaction("resourceChange", "resourceChange:metabase:apiResource");
try {
HttpApi httpApiResource = null;
if (this.queryResourceFromMetabase.booleanValue()) {
httpApiResource = (HttpApi)this.metabaseClientProxy.findByUri(apiUri, HttpApi.class);
} else {
Query query = new Query();
query.addCriteria(Criteria.where("level").is("其他"));//is:等于
// httpApiResource = this.mongoClientProxy.findApiByApiUri(apiUri);
boolean collectionExists = mongoTemplate.collectionExists(HttpApi.class);
MongoCollection<Document> collection = mongoTemplate.getCollection("httpApi");
Set<String> collectionNames = mongoTemplate.getCollectionNames();
List<HttpApi> httpApiResources = new ArrayList<>();
List<TaskInfo> taskInfoList = mongoTemplate.findAll(TaskInfo.class);
List<HttpApi> httpApis = mongoTemplate.find(query, HttpApi.class);
//while (true){
// Thread.sleep(5000);
//}
}
if (httpApiResource != null) {
JSONObject payload = JSON.parseObject(JSON.toJSONString(httpApiResource, new SerializerFeature[] { SerializerFeature.DisableCircularReferenceDetect }));
payloadMapping(apiInfos, payload, mappings);
}
t.setHandleStatus("0", (httpApiResource != null));
} catch (Exception e) {
t.error(e);
} finally {
t.complete();
}
if (this.queryApiWeaknessEnable.booleanValue()) {
Monitor.TransactionInfo t2 = Monitor.newTransaction("resourceChange", "resourceChange:mongo:apiWeakness");
try {
fillApiWeaknessInfo(mappings, apiUri, apiInfos);
t2.setHandleStatus("0", (apiInfos.get("weaknessId") != null));
} catch (Exception e) {
t2.error(e);
} finally {
t2.complete();
}
}
if (MapUtils.isNotEmpty(apiInfos)) {
List<ParseConfig> apiSessionParseConfigs = (List<ParseConfig>)apiInfos.getOrDefault("sessionParseConfigs", null);
if (CollectionUtils.isNotEmpty(apiSessionParseConfigs))
Collections.sort(apiSessionParseConfigs);
List<ParseConfig> apiUserParseConfigs = (List<ParseConfig>)apiInfos.getOrDefault("userParseConfigs", null);
if (CollectionUtils.isNotEmpty(apiUserParseConfigs))
Collections.sort(apiUserParseConfigs);
List<ParseConfig> passwordParseConfigs = (List<ParseConfig>)apiInfos.getOrDefault("passwordParseConfigs", null);
if (CollectionUtils.isNotEmpty(passwordParseConfigs))
Collections.sort(passwordParseConfigs);
List<ParseConfig> loginSuccessConfigs = (List<ParseConfig>)apiInfos.getOrDefault("loginSuccessConfigs", null);
if (CollectionUtils.isNotEmpty(loginSuccessConfigs))
Collections.sort(loginSuccessConfigs);
}
return apiInfos;
}
private Map<String, Object> loadAppInfo(String appUri) {
Map<String, Object> appInfos = new HashMap<>();
HttpAppResource httpAppResource = null;
if (this.queryResourceFromMetabase.booleanValue()) {
httpAppResource = (HttpAppResource)this.metabaseClientProxy.findByUri(appUri, HttpAppResource.class);
} else {
httpAppResource = this.mongoClientProxy.findAppByApiUri(appUri);
}
Map<String, MetabaseMappingConfig.FiledDesc> mappings = this.metabaseMappingConfig.getHttpAppMappings();
if (httpAppResource != null) {
JSONObject payload = JSON.parseObject(JSON.toJSONString(httpAppResource, new SerializerFeature[] { SerializerFeature.DisableCircularReferenceDetect }));
payloadMapping(appInfos, payload, mappings);
}
if (MapUtils.isNotEmpty(appInfos)) {
List<ParseConfig> appSessionParseConfigs = (List<ParseConfig>)appInfos.getOrDefault("sessionParseConfigs", null);
if (CollectionUtils.isNotEmpty(appSessionParseConfigs))
Collections.sort(appSessionParseConfigs);
List<ParseConfig> appUserParseConfigs = (List<ParseConfig>)appInfos.getOrDefault("userParseConfigs", null);
if (CollectionUtils.isNotEmpty(appUserParseConfigs))
Collections.sort(appUserParseConfigs);
}
return appInfos;
}
private void fillApiWeaknessInfo(Map<String, MetabaseMappingConfig.FiledDesc> mappings, String uri, Map<String, Object> resourceTagMap) {
if (!mappings.containsKey("weaknessId") && !mappings.containsKey("enumerateParaLabelConfigs"))
return;
ApiWeakness apiWeakness = this.mongoClientProxy.findApiWeakness(uri);
if (apiWeakness == null)
return;
if (mappings.containsKey("weaknessId"))
resourceTagMap.put(((MetabaseMappingConfig.FiledDesc)mappings.get("weaknessId")).getFieldName(), apiWeakness.getWeaknessId());
if (mappings.containsKey("enumerateParaLabelConfigs"))
resourceTagMap.put(((MetabaseMappingConfig.FiledDesc)mappings.get("enumerateParaLabelConfigs")).getFieldName(), getEnumerateParaLabelConfigs(apiWeakness));
}
private List<HttpApi.LabelConfig> getEnumerateParaLabelConfigs(ApiWeakness apiWeakness) {
List<Sample> samples = apiWeakness.getSamples();
if (CollectionUtils.isEmpty(samples))
return null;
List<Proof> proofs = ((Sample)samples.get(0)).getProof();
if (CollectionUtils.isEmpty(proofs))
return null;
Proof proof = proofs.get(0);
HttpApi.LabelConfig labelConfig = new HttpApi.LabelConfig();
String location = proof.getLocation().name();
labelConfig.setLocations(Lists.newArrayList(new String[] { location }));
String source = HttpEventUtils.getSource(location).name();
labelConfig.setSource(source);
labelConfig.setExtractEnable(Boolean.valueOf(true));
labelConfig.setType(proof.getExtractType().toUpperCase());
labelConfig.setLabels(Lists.newArrayList(new String[] { "enumeratePara" }));
labelConfig.setIsOnlyPathExtract(Boolean.valueOf(true));
HttpApi.LabelDetail labelDetail = new HttpApi.LabelDetail();
labelDetail.setLabelId("enumeratePara");
labelDetail.setExtractEnable(Boolean.valueOf(true));
labelConfig.setLabelDetails(Lists.newArrayList(new HttpApi.LabelDetail[] { labelDetail }));
labelConfig.setPath(proof.getPath());
return Lists.newArrayList(new HttpApi.LabelConfig[] { labelConfig });
}
public void update(ResourceChangedEvent resourceChangedEvent) {
String uri = resourceChangedEvent.getUri();
if ("httpApi".equals(resourceChangedEvent.getResourceType())) {
apiCache.put(uri, Optional.ofNullable(loadApiInfo(uri)));
} else if ("httpApp".equals(resourceChangedEvent.getResourceType())) {
appCache.put(uri, Optional.ofNullable(loadAppInfo(uri)));
}
}
private void payloadMapping(Map<String, Object> infoMap, JSONObject payload, Map<String, MetabaseMappingConfig.FiledDesc> mappings) {
mappings.forEach((source, target) -> {
// mapping中的内容和payload中基本相同;如果mapping中target存在,使用;不存在, payload中根据属性名获取target;
Object sourceValue = getValue(payload, source);
Object targetValue = null;
if (StringUtils.isEmpty(target.getClassName())) {
targetValue = sourceValue;
} else {
try {
Class<?> clazz = Class.forName(target.getClassName());
if (Boolean.TRUE.equals(target.getIsArray())) {
targetValue = JSON.parseArray(JSON.toJSONString(sourceValue, new SerializerFeature[] { SerializerFeature.DisableCircularReferenceDetect }), clazz);
} else if (clazz.getSuperclass().equals(Number.class) || clazz.equals(Boolean.class) || clazz.equals(String.class)) {
targetValue = clazz.cast(payload.get(source));
} else {
targetValue = JSON.parseObject(JSON.toJSONString(sourceValue, new SerializerFeature[] { SerializerFeature.DisableCircularReferenceDetect }), clazz);
}
} catch (Exception e) {
log.warn("metabase field parse failed, payload: {}, source: {}", new Object[] { payload, source, e });
}
}
infoMap.put(target.getFieldName(), targetValue);
});
}
private Object getValue(JSONObject payload, String path) {
JSONPath jsonPath = JSONPath.compile(path);
return jsonPath.eval(payload);
}
public void loadResources() {
loadApiResource();
loadAppResource();
try {
Thread.sleep(60000L);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
private void loadApiResource() {
try {
Map<String, MetabaseMappingConfig.FiledDesc> mappings = this.metabaseMappingConfig.getHttpApiMappings();
FindIterable<Document> findIterable = this.mongoClientProxy.getCursorFromHttpApi();
JsonWriterSettings settings = JsonWriterSettings.builder().int64Converter(new Converter<Long>() {
public void convert(Long value, StrictJsonWriter writer) {
writer.writeNumber(value.toString());
}
}).build();
for (MongoCursor<Document> mongoCursor = findIterable.iterator(); mongoCursor.hasNext(); ) {
Document document = mongoCursor.next();
Monitor.TransactionInfo t = Monitor.newTransaction("metabase", "cache:metabase:loadApi");
try {
if (document != null) {
HttpApi httpApiResource = (HttpApi)JSONObject.parseObject(document.toJson(settings), HttpApi.class);
String uri = httpApiResource.getUri();
if (StringUtils.isEmpty(uri)) {
t.complete();
continue;
}
Map<String, Object> apiInfos = new HashMap<>();
JSONObject payload = JSON.parseObject(JSON.toJSONString(httpApiResource, new SerializerFeature[] { SerializerFeature.DisableCircularReferenceDetect }));
payloadMapping(apiInfos, payload, mappings);
apiCache.put(uri, Optional.ofNullable(apiInfos));
}
t.setHandleStatus("0", true);
} catch (Exception e) {
t.error(e);
} finally {
t.complete();
}
}
} catch (Exception e) {
log.error("loadApiResource error. reason: ", e);
}
}
private void loadAppResource() {
try {
JsonWriterSettings settings = JsonWriterSettings.builder().int64Converter(new Converter<Long>() {
public void convert(Long value, StrictJsonWriter writer) {
writer.writeNumber(value.toString());
}
}).build();
Map<String, MetabaseMappingConfig.FiledDesc> mappings = this.metabaseMappingConfig.getHttpAppMappings();
FindIterable<Document> findIterable = this.mongoClientProxy.getCursorFromHttpApp();
for (MongoCursor<Document> mongoCursor = findIterable.iterator(); mongoCursor.hasNext(); ) {
Document document = mongoCursor.next();
Monitor.TransactionInfo t = Monitor.newTransaction("metabase", "cache:metabase:loadApp");
try {
HttpAppResource httpAppResource = (HttpAppResource)JSONObject.parseObject(document.toJson(settings), HttpAppResource.class);
if (httpAppResource != null) {
String uri = httpAppResource.getUri();
if (StringUtils.isEmpty(uri)) {
t.complete();
continue;
}
Map<String, Object> appInfos = new HashMap<>();
JSONObject payload = JSON.parseObject(JSON.toJSONString(httpAppResource, new SerializerFeature[] { SerializerFeature.DisableCircularReferenceDetect }));
payloadMapping(appInfos, payload, mappings);
appCache.put(uri, Optional.ofNullable(appInfos));
}
t.setHandleStatus("0", true);
} catch (Exception e) {
t.error(e);
} finally {
t.complete();
}
}
} catch (Exception e) {
log.error("loadAppResource error. reason: ", e);
}
}
}