idea环境下-使用java下载网页-爬取网页

工具函数代码

private static  String httpRequest(String requestUrl){
        StringBuffer buffer =  null;
        BufferedReader bufferedReader = null;
        InputStreamReader inputStreamReader = null;
        InputStream inputStream = null;
        HttpURLConnection httpUelConn = null;
        try {
            URL url = new URL(requestUrl);
            httpUelConn = (HttpURLConnection) url.openConnection();
            httpUelConn.setDoInput(true);
            httpUelConn.setRequestMethod("GET");
            inputStream = httpUelConn.getInputStream();
            inputStreamReader = new InputStreamReader(inputStream,"utf-8");
            bufferedReader = new BufferedReader(inputStreamReader);
            buffer = new StringBuffer();
            String str = null;
            while ((str=bufferedReader.readLine())!=null){
                buffer.append(str);
            }
        }catch (Exception e){
            e.printStackTrace();
        }finally {
            if(bufferedReader != null){
                try {
                    bufferedReader.close();
                }catch (Exception e){
                    e.printStackTrace();
                }
            }
            if(inputStreamReader != null){
                try {
                    inputStreamReader.close();
                }catch (Exception e){
                    e.printStackTrace();
                }
            }
            if(inputStream != null){
                try {
                    inputStream.close();
                }catch (Exception e){e.printStackTrace();}
            }
            if(httpUelConn != null){
                httpUelConn.disconnect();
            }
        }
        return buffer.toString();
    }

使用案例

String html = httpRequest("http://127.0.0.1:8020/YCYYWeb1/html/buffer.html?__hbt=1600846524201");