java抓取页面数据_JAVA爬虫抓取页面的URL数据

最新推荐文章于 2022-12-05 18:02:06 发布

令狐诗楠

最新推荐文章于 2022-12-05 18:02:06 发布

阅读量158

点赞数

文章标签： java抓取页面数据

本文链接：https://blog.csdn.net/weixin_35970195/article/details/114040480

版权

这个Java类主要用于从远程API获取气象数据，包括当前、2天和7天的预报。它使用了HttpClient库进行HTTP请求，解析JSON响应，并提供了转换天气状况和风向的方法。此外，还包含了对日期的处理和日志记录。

摘要由CSDN通过智能技术生成

packagecom.gsafety.langfang.screendisplay.utils;importjava.io.BufferedInputStream;importjava.io.ByteArrayOutputStream;importjava.io.IOException;importjava.io.InputStream;importjava.io.UnsupportedEncodingException;importjava.net.HttpURLConnection;importjava.net.MalformedURLException;importjava.net.URL;importjava.net.URLEncoder;importjava.text.SimpleDateFormat;importjava.util.ArrayList;importjava.util.Calendar;importjava.util.Date;importjava.util.GregorianCalendar;importjava.util.HashMap;importjava.util.List;importjava.util.Map;importjavax.annotation.Resource;importorg.apache.commons.httpclient.HttpClient;importorg.apache.commons.httpclient.methods.GetMethod;importorg.apache.commons.lang.StringUtils;importorg.apache.log4j.Logger;importorg.springframework.stereotype.Controller;importorg.springframework.web.bind.annotation.RequestMapping;importorg.springframework.web.bind.annotation.RequestMethod;importorg.springframework.web.bind.annotation.ResponseBody;importcom.google.gson.Gson;importcom.gsafety.cloudframework.common.base.page.PageResult;importcom.gsafety.cloudframework.config.util.ConfigCacheUtil;importcom.gsafety.langfang.screendisplay.vo.Returnmsg;importnet.sf.json.JSONObject;public classWeatherUtil {private static Logger logger = Logger.getLogger(WeatherUtil.class);private staticString wUrl;private staticString area;private staticString type2day;private staticString type5day;private staticString key;static{

String weatherUrl= ConfigCacheUtil.getConf("weatherUrl").getValue();if(StringUtils.isNotEmpty(weatherUrl)){

JSONObject jsonObject=JSONObject.fromObject(weatherUrl);

wUrl= jsonObject.getString("url"); //url

area = jsonObject.getString("langfangAreaCode"); //区域编码

type2day= jsonObject.getString("type2d");//2天数据类型

type5day= jsonObject.getString("type5d");//7天数据类型

key= jsonObject.getString("key");//key值

}

}private static SimpleDateFormat SDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");/*** 气象小窗口接口请求

* 数据为当前的数据

*@returnjson*/

public staticJSONObject getDataJson() {

JSONObject json= null;try{

Calendar cd=Calendar.getInstance();//开始时间

String startTime =SDF.format(cd.getTime());

logger.info("*************"+startTime +"气象接口--请求开始**********************");//http://api.weatherdt.com/common/?area=101160901&type=observe&key=fd034bf8fe70289698ec4ea79876feaa

String url = wUrl + area + "&type=" + "observe" + "&key=fd034bf8fe70289698ec4ea79876feaa";//拼装请求

GetMethod get = newGetMethod(url);

get.releaseConnection();//调用方法

HttpClient client = newHttpClient();

logger.info("*************气象接口--地址：" + url + "**********************");

String result= "";try{int executeMethod =client.executeMethod(get);

result=get.getResponseBodyAsString();

}catch(IOException e) {

e.printStackTrace();

}if(StringUtils.isEmpty(result)) {

result= "{'observe':{'101160901':{'1001002':{'006': '0.0','000':'17:15','005':'53','004':'2','003':'2','007':'879','002':'25'}}}}";

}

json=JSONObject.fromObject(result.toString());

logger.info("*************气象接口--返回值：" + result + "**********************");//结束时间

String endTime =SDF.format(cd.getTime());

logger.info("*************"+endTime+"气象接口--请求结束**********************");

}catch(Exception e) {

e.printStackTrace();

logger.info("*************气象接口--请求失败**********************");

}returnjson;

}/*** 其他气象网站的数据,因为大屏要的数据显示不完全

*@returnjson 暂时没有用,以后可以用

* **/

public staticJSONObject getDataofJson() {

JSONObject json= null;

String responseStr= null;

Map map= null;try{

Calendar cd=Calendar.getInstance();//开始时间

String endTime =SDF.format(cd.getTime());

cd.add(Calendar.DATE,-60);//结束时间

String startTime =SDF.format(cd.getTime());

logger.info("*************气象接口--请求开始**********************");//http://www.weather.com.cn/data/sk/101160901.html

URL url = new URL("http://www.weather.com.cn/data/sk/101160901.html");

logger.info("*************气象接口--地址：http://www.weather.com.cn/data/sk/101160901.html**********************");//建立http连接

HttpURLConnection conn =(HttpURLConnection) url.openConnection();//设置允许输出

conn.setDoOutput(true);

conn.setDoInput(true);//设置不用缓存

conn.setUseCaches(false);//设置传递方式

conn.setRequestMethod("GET");//设置维持长连接

conn.setRequestProperty("Connection", "Keep-Alive");//设置文件字符集:

conn.setRequestProperty("Charset", "UTF-8");//设置文件类型:

conn.setRequestProperty("contentType", "application/json");//开始连接请求

conn.connect();

logger.info("*************气象接口--状态："+conn.getResponseCode()+"**********************");//请求返回的状态

if (conn.getResponseCode() == 200) {//请求返回的数据

InputStream in =conn.getInputStream();byte[] data1 =readBig(in);

in.read(data1);//转成字符串

responseStr = new String(data1, "utf-8");//logger.info("*************气象接口--返回值："+responseStr+"**********************");

Gson gson=newGson();

json=JSONObject.fromObject(responseStr);

}else{

logger.info("*************气象接口--请求失败**********************");

}

logger.info("*************气象接口--请求结束**********************");

}catch(IOException e) {

e.printStackTrace();

}returnjson;

}/*** 气象小窗口接口请求

* 数据为2天的数据

*@returnjson

**/

public static JSONObject getData2dayJson() throwsjava.net.UnknownHostException {

JSONObject json= null;

String url= null;

Calendar cd=Calendar.getInstance();//开始时间

String startTime =SDF.format(cd.getTime());

logger.info("*************"+startTime +"气象接口--请求开始**********************");try{

url= wUrl+area+"&type=forecast"+URLEncoder.encode("[","UTF-8")+"24h_2d"+URLEncoder.encode("{","UTF-8")+"001,002"+URLEncoder.encode("}]","UTF-8")+"&key="+key;

}catch(UnsupportedEncodingException e1) {//TODO Auto-generated catch block

e1.printStackTrace();

}//拼装请求

GetMethod get = newGetMethod(url);

get.releaseConnection();//调用方法

HttpClient client = newHttpClient();

logger.info("*************气象接口--地址：" + url + "**********************");

String result= "";try{int executeMethod =client.executeMethod(get);

result=get.getResponseBodyAsString();

}catch(IOException e) {

e.printStackTrace();

logger.info("*************气象接口--请求失败**********************");

}if(StringUtils.isEmpty(result)) {

result= "{'forecast':{'24h':{'101090601':{'1001001':[{'003':'35','004':'21','001':'00','002':'00'},{'003':'35','004':'21','001':'00','002':'00'}]}}}}";

}

json=JSONObject.fromObject(result.toString());

logger.info("*************气象接口--返回值：" + result + "**********************");//结束时间+

String endTime =SDF.format(cd.getTime());

logger.info("*************"+endTime+"气象接口--请求结束**********************");returnjson;

}/*** 气象大窗口接口请求

* 数据为7天的数据

*@returnjson*/

public staticJSONObject getData7dayJson() {

JSONObject json= null;

String url=null;try{

Calendar cd=Calendar.getInstance();//开始时间

String startTime =SDF.format(cd.getTime());

logger.info("*************"+startTime +"气象接口--请求开始**********************");//URL url = new URL(wUrl+area+"&type="+"forecast[24h_5d{001,002}]&key="+key);

url = wUrl+area+"&type=forecast"+URLEncoder.encode("[","UTF-8")+"24h_5d"+URLEncoder.encode("{","UTF-8")+"001,002"+URLEncoder.encode("}]","UTF-8")+"&key="+key;

logger.info("*************气象接口--地址："+url+" **********************");//拼装请求

GetMethod get = newGetMethod(url);

get.releaseConnection();//调用方法

HttpClient client = newHttpClient();

String result= "";try{int executeMethod =client.executeMethod(get);

result=get.getResponseBodyAsString();

}catch(IOException e) {

e.printStackTrace();

}if(StringUtils.isEmpty(result)) {

result= "{'forecast':{'24h':{'101090601':{'1001001':[{'003':'35','004':'21','001':'00','002':'00'},{'003':'35','004':'21','001':'00','002':'00'},{'003':'35','004':'21','001':'00','002':'00'},{'003':'38','004':'25','001':'01','002':'01'},{'003':'36','004':'25','001':'01','002':'01'},{'003':'33','004':'23','001':'01','002':'01'},{'003':'33','004':'22','001':'01','002':'02'}]}}}}";

}

json=JSONObject.fromObject(result.toString());

logger.info("*************气象接口--结果："+result +"**********************");//结束时间+

String endTime =SDF.format(cd.getTime());

logger.info("*************"+endTime+"气象接口--请求结束**********************");

}catch(IOException e) {

e.printStackTrace();

logger.info("*************气象接口--请求失败**********************");

}returnjson;

}/*** 气象 wukaihua

* "observe": {//实况

* "101010100": {//站号

* "1001002": {//数据大类

* "006": "0",//当前降水量(单位是毫米)

* "007": "1004",//当前气压(单位百帕)

* "003": "1",//当前风力(单位是级,不用转码)

* "004": "2",//当前风向编号

* "000": "10:25",//实况发布时间

* "005": "79",//当前湿度(单位%)

* "002": "7"//当前温度(单位摄氏度)

*@return

//判断天气

public staticString getWeatherStr(String str) {if(StringUtils.isEmpty(str)) {return "";

}if ("00".equals(str)) {return "晴";

}if ("01".equals(str)) {return "多云";

}if ("02".equals(str)) {return "阴";

}//阵雨

if ("03".equals(str)) {return "阵雨";

}if ("04".equals(str)) {return "雷阵雨";

}if ("05".equals(str)) {return "雷阵雨伴有冰雹";

}if ("06".equals(str)) {return "雨夹雪";

}//小雨

if ("07".equals(str)) {return "小雨";

}//中雨

if ("08".equals(str)) {return "中雨";

}if ("21".equals(str)) {return "小到中雨";

}//大雨

if ("09".equals(str)) {return "大雨";

}if ("22".equals(str)) {return "中到大雨";

}//暴雨

if ("10".equals(str)) {return "暴雨";

}if ("11".equals(str)) {return "大暴雨";

}if ("12".equals(str)) {return "特大暴雨";

}if ("19".equals(str)) {return "冻雨";

}if ("23".equals(str)) {return "大到暴雨";

}if ("24".equals(str)) {return "暴雨到大暴雨";

}if ("25".equals(str)) {return "大暴雨到特大暴雨";

}if ("301".equals(str)) {return "雨";

}//雪

if ("13".equals(str)) {return "阵雪";

}if ("14".equals(str)) {return "小雪";

}if ("15".equals(str)) {return "中雪";

}if ("16".equals(str)) {return "大雪";

}if ("17".equals(str)) {return "暴雪";

}if ("26".equals(str)) {return "小到中雪";

}if ("27".equals(str)) {return "中到大雪";

}if ("28".equals(str)) {return "大到暴雪";

}if ("302".equals(str)) {return "雪";

}//雾

if ("18".equals(str)) {return "雾";

}if ("32".equals(str)) {return "浓雾";

}if ("49".equals(str)) {return "强浓雾";

}if ("57".equals(str)) {return "大雾";

}if ("58".equals(str)) {return "特强浓雾";

}//沙尘暴

if ("20".equals(str)) {return "沙尘暴";

}if ("29".equals(str)) {return "浮尘";

}if ("30".equals(str)) {return "扬沙";

}if ("31".equals(str)) {return "强沙尘暴";

}//霾

if ("53".equals(str)) {return "霾";

}if ("54".equals(str)) {return "中度霾";

}if ("55".equals(str)) {return "重度霾";

}if ("56".equals(str)) {return "严重霾";

}//无

if ("99".equals(str)) {return "无";

}return "";

}//判断风向

publicString getWindStr(String str) {if(StringUtils.isEmpty(str)) {return "";

}if ("0".equals(str)) {return "无持续风向";

}if ("1".equals(str)) {return "东北风";

}if ("2".equals(02)) {return "东风";

}if ("3".equals(str)) {return "东南风";

}if ("4".equals(str)) {return "南风";

}if ("5".equals(str)) {return "西南风";

}if ("6".equals(str)) {return "西风";

}if ("7".equals(str)) {return "西北风";

}if ("8".equals(str)) {return "北风";

}if ("9".equals(str)) {return "旋转风";

}return "";

}//由空气指数范围判定状态情况

public String getAirLevelStr(intstr) {if (str==0 || str>0 || str<=50) {return "优";

}if ( str>50 || str<=100) {return "良";

}if ( str>100 || str<=200) {return "轻度污染";

}if ( str>200 || str<=300) {return "中度污染";

}if (str>300) {return "重度污染";

}return "";

}//判断日期一周

public staticString getWeekOfDate(Date date) {

String[] weekDays= { "星期一", "星期二", "星期三", "星期四", "星期五", "星期六", "星期日"};

Calendar cal=Calendar.getInstance();

cal.setTime(date);int w = cal.get(Calendar.DAY_OF_WEEK) - 1;if (w < 0) {

w= 0;

}returnweekDays[w];

}private static byte[] readBig(InputStream in) throwsIOException {

BufferedInputStream bis= newBufferedInputStream(in);

ByteArrayOutputStream baos= newByteArrayOutputStream();int c =bis.read();while((c!=-1)){

baos.write(c);

c=bis.read();

}

bis.close();returnbaos.toByteArray();

}

令狐诗楠

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
复制链接

分享到 QQ

分享到新浪微博

扫一扫