爬取百度数据的java代码_Java爬取百度云观测对网站的检测数据，获取子域名及域名的安全信息...

百度云观测有对域名的检测还比较全面。下面我们来抓取下内容看看。

Java抓取代码

直接上代码，HTTP工具类暂时不能提供给您，您随便自己请求就可以了，只要能通，能返回数据。//获取子域名 & 子域名评分

public List SSLDomains(String surl){

String domain = StringUtils.getHost(surl);

surl = String.format("http://ce.baidu.com/index/getRelatedSites?site_address=%s", StringUtils.strToUrlcode(surl)) ;

List domainList = new ArrayList<>();

String res = Http.create(surl).

head("Accept", "application/json, text/javascript, */*; q=0.01")

.head("Accept-Encoding", "gzip, deflate")

.head("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")

.head("Cache-Control", "no-cache")

.head("Connection", "keep-alive")

.head("Cookie", "您的百度登录Cookie")

.head("Host", "ce.baidu.com")

.head("Pragma", "no-cache")

.head("Referer", String.format("http://ce.baidu.com/index/guance?start_url=%s", domain))

.head("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36")

.head("X-Requested-With", "XMLHttpRequest")

.timeout(3)

.readTimeout(3)

.get().send().getResponse().getResult();

if(StringUtils.isNotBlank(res) ){

JSONObject json = JSONObject.fromObject(res);

int code = json.optInt("code", 1);

if(0 == code){//成功

JSONArray data = json.optJSONArray("data");

//最多弄10个域名，有的时候子域名太多了

for (int i = 0; i < data.size() && i< 10; i++) {

Object datum = data.get(i);

JSONObject target = null;

if(datum instanceof JSONObject){

target = (JSONObject)datum;

}else{

target = JSONObject.fromObject(datum);

}

DomainCheckInfoDto.SSLDomain sslDomain = new DomainCheckInfoDto.SSLDomain(target);

domainList.add(sslDomain);

}

}

}

return domainList;

}

然后里面有一个实体类：@Data

@AllArgsConstructor

@NoArgsConstructor

public static class SSLDomain{

//分值

private int score;

//域名

private String domain;

public SSLDomain(JSONObject obj){

this.score = obj.optInt("score", 60);

this.domain = obj.optString("domain", "");

}

}

数据返回：{

"attackSSL":{

"name":"攻击风险",

"score":75,

"scoreDelta":0

},

"cateId":"medial",

"cateName":"影视音乐",

"cateScore":60,

"domainLastScore":54,

"domainSize":10,

"domsins":[

{

"domain":"air.sojson.com",

"score":84

},

{

"domain":"api.sojson.com",

"score":54

},

{

"domain":"css.sojson.com",

"score":80

},

{

"domain":"fm.sojson.com",

"score":80

},

{

"domain":"icp.sojson.com",

"score":94

},

{

"domain":"js.sojson.com",

"score":65

},

{

"domain":"m.sojson.com",

"score":80

},

{

"domain":"open.sojson.com",

"score":90

},

{

"domain":"sojson.com",

"score":90

},

{

"domain":"t.weather.sojson.com",

"score":80

}

],

"envSSL":{

"name":"网站服务",

"score":40,

"scoreDelta":0

},

"historySSL":{

"name":"网站历史安全",

"score":100,

"scoreDelta":0

},

"id":"17bb1d64a239c585",

"rank":0,

"realtimeSSL":{

"name":"网站恶意内容",

"score":75,

"scoreDelta":0

},

"synopsis":73,

"updatedTime":"2020-03-31 17:51:27"

}

这个是查询SOJSON.COM的结果

如果本文对你有帮助，那么请你赞助我，让我更有激情的写下去，帮助更多的人。

￥我需要走的更远，点击我 赞助。

如果还有疑问，点击我加群，为你提供最好的解答。