java ajax加载更多_如何用JAVA爬取AJAX加载后的页面

灵感来源(误打误撞)https://www.oschina.net/question/147939_140523

1、浏览器(google)打开要爬去的网页我这里以

http://news.iciba.com/views/dailysentence/daily.html#!/detail/title/2018-12-11

为列子爬取每日一图，一句，一翻译

如果普通爬取代码源代码是这样的显然获取不了图片url，句子，译文的所以要直接找到

请求这些参数的值的url，因为获取更多内容是通过JSONP的方式，其本质就是动态的

加载js文件并执行其中的代码以获得服务器端返回的内容。

如下：

2、打开开发则工具，点击Network，点击js按图操作

再点击Headers获取url

然后java代码实现返回的值就是我们要的东西如下图

//https://www.oschina.net/question/147939_140523

//callback

import java.io.BufferedReader;

import java.io.InputStreamReader;

import java.net.HttpURLConnection;

import java.net.URL;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

public class AnalyzeWeb {

public static void main(String args[]) {

//解析目标网址

String day = "2018-12-09";

String postfix = "&_=1544530355789";

String url = "http://sentence.iciba.com/index.php?

callback=jQuery1900046932545628676436_1544530355780&c=dailysentence&m=

getdetail&title=";

url = url + day + postfix;

String webContent = getWebTxt(url);

System.out.println(webContent);

String pattern = "content\":\"[\\w*\\s+(,)(.)]*";

takeWebDetail(webContent, pattern);

pattern = "note\":\"[\\\\\\/\\w*(.)]*";

takeWebDetail(webContent, pattern);

pattern = "translation\":\"[\\\\\\/\\w*(.)]*";

takeWebDetail(webContent, pattern);

pattern = "picture\":\"http:[\\\\\\/\\w*(.)]*";

takeWebDetail(webContent, pattern);

}

public static String getWebTxt(String url) {

URL u;

HttpURLConnection httpURLConnection;

String ret = "";

try {

u = new URL(url);

httpURLConnection = (HttpURLConnection)u.openConnection();

if (httpURLConnection.getResponseCode() == 200) {

BufferedReader bufferedReader = new BufferedReader(new

InputStreamReader(httpURLConnection.getInputStream(),"utf-8"));

String read;

while ((read = bufferedReader.readLine()) != null) {

ret += read ;

ret += "\r\n" ;

}

}

} catch (Exception e){

System.out.println("解析网页错误!");

}

return ret ;

}

public static String takeWebDetail(String str, String pattern) {

Pattern r = Pattern.compile(pattern);

Matcher m = r.matcher(str);

if (m.find( )) {

System.out.println("Found value: " + m.group(0) );

} else {

System.out.println("NO MATCH");

}

return "";

}

}