Java使用ＵＲＬ获取网页内容

使用URLConnection来获取网页的内容，发送get方法，如果所提供的是CSS代码，需求所需要的结果需要用到正则表达式来获取。

package Get;

import Post.PostMethod;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;

/**
 * 这是一个get方法,当客户端需要通过get 方法获取链接的时候就要采用这个方法
 * Created by syb on 2016/10/29.
 */
public class GetMethod {
    /**
     * 测试get 方法是否可用采用http://news.sina.com.cn/
     *
     * @param args
     */
    public static void main(String[] args) {
       /* String urlStr = "http://www.kuwo.cn/static/js/common/player/player.js?v20161009.js";
        System.out.println(sendGet(urlStr));*/
        /**
         * 采用正则表达式获取所响应的链接
         */
        //region 测试get方法
       /* String res = sendGet(urlStr);
        String regex = "http:.{4}(\\w*.){4}(.\\w*){10}.shtml";
        Pattern pattern = Pattern.compile(regex);
        Matcher m = pattern.matcher(res);
        while (m.find()){
            System.out.println(m.group());
        }*/

      //  System.out.println(res);
        //endregion
    }

    /**
     * 这是一个get方法，如果网页需要用到get来访问源码，就应该用此方法
     *
     * @param url 需要提供一个网页的链接
     * @return 返回服务器所响应的内容
     */
    public static String sendGet(String url) {
        String result = "";
        BufferedReader in = null;
        String strUrlName = url;
        try {
            URL readUrl = new URL(strUrlName);
            URLConnection urlConection = readUrl.openConnection();
            //设置属性
         //   urlConection.setRequestProperty("Accept", "***/*//*//**//*");
           /* urlConection.setRequestProperty("Connection", "keep-alive");
            urlConection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0");*/
            PostMethod p = new PostMethod();
            urlConection = p.urlConnectionSetParams(readUrl);
            urlConection.connect();
            in = new BufferedReader(new InputStreamReader(urlConection.getInputStream(), "utf-8"));
            //读取响应
            //处理响应
            String line = null;
            while ((line = in.readLine()) != null) {
                result += line;
            }
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                if (in != null) {
                    in.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        return result;
    }
}

发送post方法来获取内容

/**
 * 这是一个post的方法，当获取网页的时候如果是post方法就需要调用此方法
 *
 * @param url    想要访问的链接
 * @param params
 * @return 所获取的响应的内容
 */
public static String sendPost(String url, String ...params) {
    String result = null;
    PrintWriter out = null; //用来读取响应
    BufferedReader in = null; //将响应存储
    try {
        URL realUrl = new URL(url);
        URLConnection urlConnection = null;
        urlConnection = urlConnectionSetParams(realUrl);
        urlConnection.setDoInput(true);
        urlConnection.setDoOutput(true);//设置属性
        urlConnection.connect();
        //获取输出流
        out = new PrintWriter(urlConnection.getOutputStream());
        //发送请求
        out.print(params);
        out.flush();
        //已经发送请求，接下来开始获取响应
        in = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), "utf-8"));
        //读取响应里面的内容
        String line = null;
        while ((line = in.readLine()) != null) {
            result += line;
        }
    } catch (MalformedURLException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        try {
            if (in != null) {
                in.close();
            }
            if (out != null) {
                out.close();
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    return result;
}

/**
 * 这个方法是设置浏览器的属性
 *
 * @param rl   要访问的链接
 * 返回一个URLConnection的对象
 * @return
 */
public static URLConnection urlConnectionSetParams(URL rl) {
    try {
        URLConnection urlConnection = rl.openConnection();
        //设置通用请求
        urlConnection.setRequestProperty("accept", "*/*");
        urlConnection.setRequestProperty("connection", "keep-alive");
        urlConnection.setRequestProperty("user-agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0");
        urlConnection.setRequestProperty("Upgrade-Insecure-Requests","1");
       // urlConnection.setRequestProperty("Accept-Encoding","gzip");
        //urlConnection.setRequestProperty("Host","gsxt.xjaic.gov.cn:7001");

        return urlConnection;
    } catch (IOException e) {
        e.printStackTrace();
        return null;
    }
}

}

“`
以上仅为一种获取链接的方法