某项目运行在政务网,涉及的照片、视频等数据存储在内部华为云。同时该项目也可以在互联网访问。这样子的话,照片、视频就有个安全性问题。抛开保密性不谈,一个视频动辄几个G,如果它的链接在互联网上长久有效,想播放就能播放,哪里有这么多带宽消耗得起。
老实说,华为云在这方面处理得还可以,尽管对开发者不够友好。主要是帮助文档说得不清不楚,说一点漏一点,又缺乏示例,只能靠程序员苦苦探寻。不过,几乎所有得帮助文档都这个鬼样。
一、安全措施
1、对象
对于存储在华为云中的对象来说,安全性措施就是提供一个临时的链接。所谓临时,就是有时效性,比如30分钟后就失效;同时链接会带上签名,签名根据密钥和一定的算法计算所得,与链接里面的元素互相印证。如果没有签名,对象将拒绝访问。
这种做法,跟JWT(JSON WEB TOKEN)认证原理是一样的。主要部分就是所谓的签名。这个签名,是将请求地址里的信息,比如时间戳、,用密钥等计算,生成一个签名(就是一个摘要),然后附在地址后面,一起发给服务器。服务器收到后,用同样的方法进行计算,得到摘要,两相比较,相符即认证通过。
2、桶或文件夹
在相关文章里说到,桶有公共只读、私有等等属性;然后又有所谓桶访问策略,可以对桶里的对象做各种访问控制。分配给我们的政务网华为云,桶私有,然后访问策略禁止匿名访问。最直观的体验,就是一个黑匣子。该桶里面有什么文件夹,不知道;每个文件夹里有什么文件,不知道;直接指定其中一个文件访问,拒绝。完全是狗咬乌龟,无处下牙。
二、获得对象链接
代码基本上抄自华为云的官方文档 URL中携带签名
1、pom.xml
<dependency>
<groupId>dom4j</groupId>
<artifactId>dom4j</artifactId>
<version>1.6.1</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.1.41</version>
</dependency>
2、华为云密钥及一些常量
就是ak、sk、端点等信息。端点就是桶地址,桶和华为云域名的结合。
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
public class Sign {
protected static final String SIGN_SEP = "\n";
protected static final String OBS_PREFIX = "x-obs-";
protected static final String DEFAULT_ENCODING = "UTF-8";
protected static final List<String> SUB_RESOURCES = Collections.unmodifiableList(Arrays.asList(
"CDNNotifyConfiguration", "acl", "append", "attname", "backtosource", "cors", "customdomain", "delete",
"deletebucket", "directcoldaccess", "encryption", "inventory", "length", "lifecycle", "location", "logging",
"metadata", "mirrorBackToSource", "modify", "name", "notification", "obscompresspolicy", "orchestration",
"partNumber", "policy", "position", "quota", "rename", "replication", "response-cache-control",
"response-content-disposition", "response-content-encoding", "response-content-language", "response-content-type",
"response-expires", "restore", "storageClass", "storagePolicy", "storageinfo", "tagging", "torrent", "truncate",
"uploadId", "uploads", "versionId", "versioning", "versions", "website", "x-image-process",
"x-image-save-bucket", "x-image-save-object", "x-obs-security-token", "object-lock", "retention"));
protected String ak = "app key";
protected String sk = "security key";
protected String bucketName = "桶名";
protected String endpoint = "华为云域名(含桶名),如 bucketname.obs.cn-north-4.myhuaweicloud.com";
public String getEndpoint() {
return endpoint;
}
}
3、生成对象临时链接类
这个类只有一个public方法,作用就是根据对象名称和过期时间,结合ak、sk等信息,算出临时链接串。
import org.springframework.stereotype.Component;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.regex.Pattern;
@Component
public class SignUrl extends Sign {
public String getURL(String objectName, long expires) throws Exception {
Map<String, String[]> headers = new HashMap<String, String[]>();
Map<String, String> queries = new HashMap<String, String>();
String signature = querySignature("GET", headers, queries, bucketName, objectName, expires);
return getURL(queries,objectName,signature,expires);
}
/* 以下都是私有函数 */
private String getURL(Map<String, String> queries, String objectName, String signature, long expires) throws UnsupportedEncodingException {
StringBuilder URL = new StringBuilder();
URL.append("https://").append(this.host).append("/").
append(this.encodeObjectName(objectName)).append("?");
String key;
for (Map.Entry<String, String> entry : queries.entrySet()) {
key = entry.getKey();
if (key == null) {
continue;
}
if (SUB_RESOURCES.contains(key)) {
String value = entry.getValue();
URL.append(key);
if (value != null) {
URL.append("=").append(value).append("&");
} else {
URL.append("&");
}
}
}
URL.append("AccessKeyId=").append(this.ak).append("&Expires=").append(expires).
append("&Signature=").append(signature);
return URL.toString();
}
private boolean isBucketNameValid(String bucketName) {
if (bucketName == null || bucketName.length() > 63 || bucketName.length() < 3) {
return false;
}
if (!Pattern.matches("^[a-z0-9][a-z0-9.-]+$", bucketName)) {
return false;
}
if (Pattern.matches("(\\d{1,3}\\.){3}\\d{1,3}", bucketName)) {
return false;
}
String[] fragments = bucketName.split("\\.");
for (int i = 0; i < fragments.length; i++) {
if (Pattern.matches("^-.*", fragments[i]) || Pattern.matches(".*-$", fragments[i])
|| Pattern.matches("^$", fragments[i])) {
return false;
}
}
return true;
}
private String encodeUrlString(String path) throws UnsupportedEncodingException {
return URLEncoder.encode(path, DEFAULT_ENCODING)
.replaceAll("\\+", "%20")
.replaceAll("\\*", "%2A")
.replaceAll("%7E", "~");
}
private String encodeObjectName(String objectName) throws UnsupportedEncodingException {
StringBuilder result = new StringBuilder();
String[] tokens = objectName.split("/");
for (int i = 0; i < tokens.length; i++) {
result.append(this.encodeUrlString(tokens[i]));
if (i < tokens.length - 1) {
result.append("/");
}
}
return result.toString();
}
private String join(List<?> items, String delimiter) {//delimiter,定界符
StringBuilder sb = new StringBuilder();
for (int i = 0; i < items.size(); i++) {
String item = items.get(i).toString();
sb.append(item);
if (i < items.size() - 1) {
sb.append(delimiter);
}
}
return sb.toString();
}
private boolean isValid(String input) {
return input != null && !input.equals("");
}
private String hmacSha1(String input) throws NoSuchAlgorithmException, InvalidKeyException, UnsupportedEncodingException {
SecretKeySpec signingKey = new SecretKeySpec(this.sk.getBytes(DEFAULT_ENCODING), "HmacSHA1");
Mac mac = Mac.getInstance("HmacSHA1");
mac.init(signingKey);
return Base64.getEncoder().encodeToString(mac.doFinal(input.getBytes(DEFAULT_ENCODING)));
}
private String stringToSign(String httpMethod, Map<String, String[]> headers, Map<String, String> queries,
String bucketName, String objectName) throws Exception {
String contentMd5 = "";
String contentType = "";
String date = "";
TreeMap<String, String> canonicalizedHeaders = new TreeMap<String, String>();
String key;
List<String> temp = new ArrayList<String>();
for (Map.Entry<String, String[]> entry : headers.entrySet()) {
key = entry.getKey();
if (key == null || entry.getValue() == null || entry.getValue().length == 0) {
continue;
}
key = key.trim().toLowerCase(Locale.ENGLISH);
if (key.equals("content-md5")) {
contentMd5 = entry.getValue()[0];
continue;
}
if (key.equals("content-type")) {
contentType = entry.getValue()[0];
continue;
}
if (key.equals("date")) {
date = entry.getValue()[0];
continue;
}
if (key.startsWith(OBS_PREFIX)) {
for (String value : entry.getValue()) {
if (value != null) {
temp.add(value.trim());
}
}
canonicalizedHeaders.put(key, this.join(temp, ","));
temp.clear();
}
}
if (canonicalizedHeaders.containsKey("x-obs-date")) {
date = "";
}
// handle method/content-md5/content-type/date
StringBuilder stringToSign = new StringBuilder();
stringToSign.append(httpMethod).append(SIGN_SEP)
.append(contentMd5).append(SIGN_SEP)
.append(contentType).append(SIGN_SEP)
.append(date).append(SIGN_SEP);
// handle canonicalizedHeaders
for (Map.Entry<String, String> entry : canonicalizedHeaders.entrySet()) {
stringToSign.append(entry.getKey()).append(":").append(entry.getValue()).append(SIGN_SEP);
}
// handle CanonicalizedResource
stringToSign.append("/");
if (this.isValid(bucketName)) {
stringToSign.append(bucketName).append("/");
if (this.isValid(objectName)) {
stringToSign.append(this.encodeObjectName(objectName));
}
}
TreeMap<String, String> canonicalizedResource = new TreeMap<String, String>();
for (Map.Entry<String, String> entry : queries.entrySet()) {
key = entry.getKey();
if (key == null) {
continue;
}
if (SUB_RESOURCES.contains(key)) {
canonicalizedResource.put(key, entry.getValue());
}
}
if (canonicalizedResource.size() > 0) {
stringToSign.append("?");
for (Map.Entry<String, String> entry : canonicalizedResource.entrySet()) {
stringToSign.append(entry.getKey());
if (this.isValid(entry.getValue())) {
stringToSign.append("=").append(entry.getValue());
}
stringToSign.append("&");
}
stringToSign.deleteCharAt(stringToSign.length() - 1);
}
// System.out.println(String.format("StringToSign:%s%s", SIGN_SEP, stringToSign.toString()));
return stringToSign.toString();
}
private String querySignature(String httpMethod, Map<String, String[]> headers, Map<String, String> queries,
String bucketName, String objectName, long expires) throws Exception {
if (!isBucketNameValid(bucketName)) {
throw new IllegalArgumentException("the bucketName is illegal");
}
if (headers.containsKey("x-obs-date")) {
headers.put("x-obs-date", new String[]{String.valueOf(expires)});
} else {
headers.put("date", new String[]{String.valueOf(expires)});
}
//1. stringToSign
String stringToSign = this.stringToSign(httpMethod, headers, queries, bucketName, objectName);
//2. signature
return this.encodeUrlString(this.hmacSha1(stringToSign));
}
}
三、列举桶内对象
这个类比上面的复杂。当然绝大部分也都是抄自华为云官网。列举桶内对象
注意没有一个所谓列举文件夹内对象的概念。要列举文件夹内对象,需要用prefix参数,以这样的格式:
https://endpoint?prefix=文件夹 ,比如
https://bucketname1.obs.cn-north-4.myhuaweicloud.com/?prefix=hadyx/1dssp/
hadyx是桶bucketname1下的文件夹,然后ldssp是hadyx的子文件夹。
较为复杂的地方,在于如何生成签名。生成签名的算法是固定的,关键是要传递些什么参数去生成这个签名,关于列举桶内对象,生成签名所需传递的参数,官方文档语焉不详;再一个,生成对象的临时链接,纯粹计算得到一个字符串,返回即可;而列举桶内对象,需要将签名等附在请求头里,访问华为云,然后获得结果。
import org.springframework.stereotype.Component;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.*;
@Component
public class SignHead extends Sign {
public String urlEncode(String input) throws UnsupportedEncodingException
{
return URLEncoder.encode(input, DEFAULT_ENCODING)
.replaceAll("%7E", "~") //for browser
.replaceAll("%2F", "/")
.replaceAll("%20", "+");
}
private String join(List<?> items, String delimiter)
{
StringBuilder sb = new StringBuilder();
for (int i = 0; i < items.size(); i++)
{
String item = items.get(i).toString();
sb.append(item);
if (i < items.size() - 1)
{
sb.append(delimiter);
}
}
return sb.toString();
}
private boolean isValid(String input) {
return input != null && !input.equals("");
}
public String hamcSha1(String input) throws NoSuchAlgorithmException, InvalidKeyException, UnsupportedEncodingException {
SecretKeySpec signingKey = new SecretKeySpec(this.sk.getBytes(DEFAULT_ENCODING), "HmacSHA1");
Mac mac = Mac.getInstance("HmacSHA1");
mac.init(signingKey);
return Base64.getEncoder().encodeToString(mac.doFinal(input.getBytes(DEFAULT_ENCODING)));
}
private String stringToSign(String httpMethod, Map<String, String[]> headers, Map<String, String> queries,
String bucketName, String objectName) throws Exception{
String contentMd5 = "";
String contentType = "";
String date = "";
TreeMap<String, String> canonicalizedHeaders = new TreeMap<String, String>();
String key;
List<String> temp = new ArrayList<String>();
for(Map.Entry<String, String[]> entry : headers.entrySet()) {
key = entry.getKey();
if(key == null || entry.getValue() == null || entry.getValue().length == 0) {
continue;
}
key = key.trim().toLowerCase(Locale.ENGLISH);
if(key.equals("content-md5")) {
contentMd5 = entry.getValue()[0];
continue;
}
if(key.equals("content-type")) {
contentType = entry.getValue()[0];
continue;
}
if(key.equals("date")) {
date = entry.getValue()[0];
continue;
}
if(key.startsWith(OBS_PREFIX)) {
for(String value : entry.getValue()) {
if(value != null) {
temp.add(value.trim());
}
}
canonicalizedHeaders.put(key, this.join(temp, ","));
temp.clear();
}
}
if(canonicalizedHeaders.containsKey("x-obs-date")) {
date = "";
}
// handle method/content-md5/content-type/date
StringBuilder stringToSign = new StringBuilder();
stringToSign.append(httpMethod).append(SIGN_SEP)
.append(contentMd5).append(SIGN_SEP)
.append(contentType).append(SIGN_SEP)
.append(date).append(SIGN_SEP);
// handle canonicalizedHeaders
for(Map.Entry<String, String> entry : canonicalizedHeaders.entrySet()) {
stringToSign.append(entry.getKey()).append(":").append(entry.getValue()).append(SIGN_SEP);
}
// handle CanonicalizedResource
stringToSign.append("/");
if(this.isValid(bucketName)) {
stringToSign.append(bucketName).append("/");
if(this.isValid(objectName)) {
stringToSign.append(this.urlEncode(objectName));
}
}
TreeMap<String, String> canonicalizedResource = new TreeMap<String, String>();
for(Map.Entry<String, String> entry : queries.entrySet()) {
key = entry.getKey();
if(key == null) {
continue;
}
if(SUB_RESOURCES.contains(key)) {
canonicalizedResource.put(key, entry.getValue());
}
}
if(canonicalizedResource.size() > 0) {
stringToSign.append("?");
for(Map.Entry<String, String> entry : canonicalizedResource.entrySet()) {
stringToSign.append(entry.getKey());
if(this.isValid(entry.getValue())) {
stringToSign.append("=").append(entry.getValue());
}
stringToSign.append("&");
}
stringToSign.deleteCharAt(stringToSign.length()-1);
}
// System.out.println(String.format("StringToSign:%s%s", SIGN_SEP, stringToSign.toString()));
return stringToSign.toString();
}
public String headerSignature(String httpMethod, Map<String, String[]> headers, Map<String, String> queries,
String objectName) throws Exception {
//1. stringToSign
String stringToSign = this.stringToSign(httpMethod, headers, queries, this.bucketName, objectName);
//2. signature
return String.format("OBS %s:%s", this.ak, this.hamcSha1(stringToSign));
}
public String querySignature(String httpMethod, Map<String, String[]> headers, Map<String, String> queries,
String bucketName, String objectName, long expires) throws Exception {
if(headers.containsKey("x-obs-date")) {
headers.put("x-obs-date", new String[] {String.valueOf(expires)});
}else {
headers.put("date", new String[] {String.valueOf(expires)});
}
//1. stringToSign
String stringToSign = this.stringToSign(httpMethod, headers, queries, bucketName, objectName);
//2. signature
return this.urlEncode(this.hamcSha1(stringToSign));
}
}
四、包装成服务方便调用
为方便调用,将上面的两个类包装成服务。
1、服务接口
public interface HwCloudService {
String getURL(String objectName, long expires) throws Exception;//获取对象临时链接
String getList(String folder) throws Exception;//列举文件夹对象
}
2、服务实现
import com.opencloud.main.server.service.HwCloudService;
import com.opencloud.main.server.utlis.SignHead;
import com.opencloud.main.server.utlis.SignUrl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.net.ssl.*;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
import java.text.SimpleDateFormat;
import java.util.*;
@Service
public class HwCloudServiceImplement implements HwCloudService {
@Autowired
private SignUrl signUrl;
@Autowired
private SignHead signHead;
/*
objectName,对象名,即文件名(不包含文件夹)
*/
@Override
public String getURL(String objectName, long expires) throws Exception {
return this.signUrl.getURL(objectName, expires);
}
/*
folder,文件夹在桶内的相对路径。如 hadyx/广州市/番禺区/030/
*/
@Override
public String getList(String folder) throws Exception {
String strDate = getStrDate();
String authorization = getListSign(folder, strDate);
URL url = getListUrl(folder);
System.out.println("url:");
System.out.println(url);
TrustHttpsConnection();//华为云为https,但内部域名的证书好像系统不认识,这里设置忽略证书问题
HttpsURLConnection con = (HttpsURLConnection) url.openConnection();
con.setRequestProperty("Authorization", authorization);
con.setRequestProperty("Date", strDate);
con.setRequestMethod("GET");
BufferedReader in = new BufferedReader(
new InputStreamReader(con.getInputStream()));
String inputLine;
StringBuffer content = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
content.append(inputLine);
}
in.close();
con.disconnect();
return content.toString();
}
private void TrustHttpsConnection() throws NoSuchAlgorithmException, KeyManagementException {
TrustManager[] trustAllCerts = new TrustManager[]{
new X509TrustManager() {
public java.security.cert.X509Certificate[] getAcceptedIssuers() {
return null;
}
public void checkClientTrusted(X509Certificate[] certs, String authType) {
}
public void checkServerTrusted(X509Certificate[] certs, String authType) {
}
}
};
SSLContext sc = SSLContext.getInstance("SSL");
sc.init(null, trustAllCerts, new java.security.SecureRandom());
HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
HostnameVerifier allHostsValid = new HostnameVerifier() {
public boolean verify(String hostname, SSLSession session) {
return true;
}
};
// set the allTrusting verifier
HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid);
}
private URL getListUrl(String folder) throws MalformedURLException {
//URL url = new URL("https://bucketname.obs.cn-north-4.myhuaweicloud.com/?prefix=hadyx/1dssp/");
return new URL(String.format("https://%s/?prefix=%s", signHead.getEndpoint(), folder));
}
private String getStrDate() {
SimpleDateFormat rfc1123 = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.US);
rfc1123.setTimeZone(TimeZone.getTimeZone("GMT"));
return rfc1123.format(new Date());
}
/*
计算列举文件夹对象签名。传入文件夹和当前时间。注意时间的格式是rfc1123
*/
private String getListSign(String folder, String strDate) throws Exception {
//桶名为空,这很重要。原因不清除,可能是域名里已经带有桶名
String objectName = "";//看来在华为云的世界中,文件夹不算对象,只有文件才算对象。
Map<String, String[]> headers = new HashMap<String, String[]>();
headers.put("date", new String[]{strDate});
Map<String, String> queries = new HashMap<String, String>();
//queries.put("prefix", "temp/");
//queries.put("prefix", "hadyx/1dssp/");
queries.put("prefix", folder);
return signHead.headerSignature("GET", headers, queries, objectName);
}
}
五、对外API
服务供项目内部调用;还可以暴露接口供前端调用
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.opencloud.common.model.ResultBody;
import com.opencloud.main.server.service.impl.HwCloudServiceImplement;
import com.opencloud.main.server.utlis.XmlTool;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@RestController
@RequestMapping("obs")
public class ObsController {
@Autowired
private HwCloudServiceImplement hwcService;
@PostMapping("/obj/one")
public String getObjUrl(@RequestParam Map<String, Object> map) {
/*
{
"fileName": "temp/test.txt",
"minutes": 30
}
*/
int minutes = Integer.parseInt(map.get("minutes").toString());//data.getMinutes();
long expires = getExpires(minutes);
String fileName = map.get("fileName").toString();
if(fileName == null || fileName.endsWith("/")){//以“/"结尾代表文件夹
return ResultBody.failed().msg("没有指定文件");
}
System.out.println(fileName);
if(fileName.startsWith("/")) fileName = fileName.substring(1);
String url = null;
try {
url = hwcService.getURL(fileName, expires);
} catch (Exception e) {
e.printStackTrace();
}
return url;
}
@PostMapping("/obj/list")
public List<String> getObjList(@RequestParam Map<String, Object> map) throws Exception {
/*
{
"folderName": "hadyx/广州市/番禺区/030/",
"minutes": 30
}
*/
List<String> lis = new ArrayList<>();
int minutes = Integer.parseInt(map.get("minutes").toString());//data.getMinutes();
long expires = getExpires(minutes);
String folder = map.get("folderName").toString();//data.getFolderName();
if(folder == null || !folder.endsWith("/")){
return ResultBody.failed().msg("没有指定文件夹");
}
System.out.println(folder);
if(folder.startsWith("/")) folder = folder.substring(1);
//返回结果为XML,转为json,方便使用
String xml = hwcService.getList(folder);
JSONObject jObj = XmlTool.documentToJSONObject(xml);
if(jObj.containsKey("Contents")) {
JSONArray jArray = jObj.getJSONArray("Contents");
Iterator<Object> it = jArray.iterator();
while (it.hasNext()) {
JSONObject jsonObj = (JSONObject) it.next();
String name = jsonObj.getString("Key");
if(!name.endsWith("/")) {
lis.add(hwcService.getURL(name, expires));
}
}
}
return lis;
}
private long getExpires(int minutes){
if(minutes > 60 * 24) minutes = 60 * 24;//最长不能超过24小时
return (System.currentTimeMillis() + minutes * 60 * 1000) / 1000;
}
}
五、华为云与minIO
minIO据说是对象存储的鼻祖。可能是真的。今天在别的项目,用到minIO进行类似操作,发现桶策略之类都很像。当然,华为云自己修修补补,也是有可能的。
相关文章:
政务网中使用内部华为云