我没有找到适用于Jsoup的标准解决方案.这是我使用HashMap进行的简单cookie处理.它可能缺少一些功能但我希望它对我的基本爬虫工作得很好:
private static HashMap> host2cookies = new HashMap>();
public static String[] DownloadPage(URL url) throws Exception
{
Connection con = Jsoup.connect(url.toString()).timeout(600000);
loadCookiesByHost(url, con);
Document doc = con.get();
url = con.request().url();
storeCookiesByHost(url, con);
return new String[]{url.toString(), doc.html()};
}
private static void loadCookiesByHost(URL url, Connection con) {
try {
String host = url.getHost();
if (host2cookies.containsKey(host)) {
HashMap cookies = host2cookies.get(host);
for (Entry cookie : cookies.entrySet()) {
con.cookie(cookie.getKey(), cookie.getValue());
}
}
} catch (Throwable t) {
// MTMT move to log
System.err.println(t.toString()+":: Error loading cookies to: " + url);
}
}
private static void storeCookiesByHost(URL url, Connection con) {
try {
String host = url.getHost();
HashMap cookies = host2cookies.get(host);
if (cookies == null) {
cookies = new HashMap();
host2cookies.put(host, cookies);
}
cookies.putAll(con.response().cookies());
} catch (Throwable t) {
// MTMT move to log
System.err.println(t.toString()+":: Error saving cookies from: " + url);
}
}