Caused by: java.lang.RuntimeException: javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
该报错问题 是证书问题导致。
使用HttpURLConnection访问https协议请求时.对SSL信任
参考链接:https://blog.csdn.net/zz153417230/article/details/80271155
package novel.spider.impl;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.TrustManager;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import novel.spider.entitys.Chapter;
import novel.spider.interfaces.IChapterSpider;
import novel.spider.junit.MyX509TrustManager;
public class AbstractChapterSpider implements IChapterSpider {
protected String crawl(String url) throws Exception {
SSLContext sslcontext = SSLContext.getInstance("SSL", "SunJSSE");//第一个参数为协议,第二个参数为提供者(可以缺省)
TrustManager[] tm = {new MyX509TrustManager()};
sslcontext.init(null, tm, new SecureRandom());
HostnameVerifier ignoreHostnameVerifier = new HostnameVerifier() {
public boolean verify(String s, SSLSession sslsession) {
System.out.println("WARNING: Hostname is not matched for cert.");
return true;
}
};
HttpsURLConnection.setDefaultHostnameVerifier(ignoreHostnameVerifier);
HttpsURLConnection.setDefaultSSLSocketFactory(sslcontext.getSocketFactory());
URL url2 = new URL(url);
HttpURLConnection conn = (HttpURLConnection) url2.openConnection();
InputStream in = (InputStream) conn.getInputStream();
// String encoding = conn.getContentEncoding();
// encoding = encoding == null ? "UTF-8" : encoding;
String resp="";
java.io.BufferedReader breader = new BufferedReader(
new InputStreamReader(in, "UTF-8"));
String str = breader.readLine();
while (str != null) {
resp+=str;
str= breader.readLine();
}
return resp;
}
@Override
public List<Chapter> getsChapter(String url) {
try {
String result = crawl(url);
Document doc = Jsoup.parse(result);
//System.err.println(doc);
Elements as = doc.select("div li a");
List<Chapter> chapters = new ArrayList<>();
for (Element a : as) {
Chapter chapter = new Chapter();
chapter.setTitle(a.text());
chapter.setUrl("http://www.bxwx8.org" + a.attr("href"));
chapters.add(chapter);
}
return chapters;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
package novel.spider.junit;
import java.util.List;
import org.junit.Test;
import novel.spider.entitys.Chapter;
import novel.spider.impl.DefaultChapterSpider;
import novel.spider.interfaces.IChapterSpider;
public class Testcase {
@Test
public void test1() throws Exception {
IChapterSpider spider = new DefaultChapterSpider();
List<Chapter> chapters = spider.getsChapter("https://www.266ks.com/0_5/");
for (Chapter chapter : chapters) {
System.out.println(chapter);
}
}
}