最近要将JSON网页数据保存下来,找到了两种形式:
第一种:模拟get请求获取数据,按实际服务器端要求选用 Post 或 Get 请求方式
HttpClient httpClient = new HttpClient();
GetMethod getMethod = new GetMethod(dataUrlLocation);
// getMethod.addRequestHeader("Accept", "*/*");
// getMethod.addRequestHeader("Connection", "keep-alive");
//设置格式为json
getMethod.addRequestHeader("Content-Type", "application/json; charset=UTF-8");
// getMethod.addRequestHeader("Cookie", cookies[cookies.length-1].toString());
// getMethod.addRequestHeader("Referer", "https://data.okc.gov/portal/page/bingmap?datasetName=Work%20Zones&mapScale=11");
// getMethod.addRequestHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3573.0 Safari/537.36");
// getMethod.addRequestHeader("X-Requested-With", "XMLHttpRequest");
httpClient.executeMethod(getMethod); //模拟get请求
String text = getMethod.getResponseBodyAsString();
int BUF_SIZE = 1024; //写入文件
String unzipped_file = DATA_FILE;
out_stream = new BufferedOutputStream(
new FileOutputStream(unzipped_file), BUF_SIZE);
byte[] input_buffer = new byte[BUF_SIZE];
int byteLength = 0;
input_buffer = text.getBytes();
byteLength = input_buffer.length;
out_stream.write(input_buffer, 0, byteLength);
第二种:模拟浏览器发送请求
HttpsURLConnection conn = (HttpsURLConnection) new URL(dataUrlLocation).openConnection();
// conn.setSSLSocketFactory(new TCITLSSocketConnectionFactory());
conn.setConnectTimeout((int) retryWaitTime);
conn.setReadTimeout((int) retryWaitTime);
conn.setDoInput(true);
conn.setDoOutput(true);
conn.setRequestProperty("Accept", "*/*");
// conn.setRequestProperty("Accept-Encoding", "gzip, deflate, br");
// conn.setRequestProperty("Accept-Language", "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7");
conn.setRequestProperty("Connection", "keep-alive");
// conn.setRequestProperty("Content-Length", param.getBytes().length + "");
conn.setRequestProperty("Content-Type", "application/json; charset=UTF-8");
conn.setRequestProperty("Cookie", "cookies[cookies.length-1]");
// conn.setRequestProperty("Host", "transview.org");
// conn.setRequestProperty("Origin", "https://transview.org");
conn.setRequestProperty("Referer", "https://data.okc.gov/portal/page/bingmap?datasetName=Work%20Zones&mapScale=11");
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3573.0 Safari/537.36");
conn.setRequestProperty("X-Requested-With", "XMLHttpRequest");
InputStream content = (InputStream) conn.getContent();
int BUF_SIZE = 1024;
String unzipped_file = DATA_FILE;
out_stream = new BufferedOutputStream(
new FileOutputStream(unzipped_file), BUF_SIZE);
byte[] input_buffer = new byte[BUF_SIZE];
int byteLength = 0;
while ((byteLength = content.read(input_buffer, 0, BUF_SIZE)) > 0) {
out_stream.write(input_buffer, 0, byteLength);
}