java 获取网页编码,如何从Java获取网页的源代码

I just want to retrieve any web page's source code from Java. I found lots of solutions so far, but I couldn't find any code that works for all the links below:

The main problem for me is that some codes retrieve web page source code, but with missing ones. For example the code below does not work for the first link.

InputStream is = fURL.openStream(); //fURL can be one of the links above

BufferedReader buffer = null;

buffer = new BufferedReader(new InputStreamReader(is, "iso-8859-9"));

int byteRead;

while ((byteRead = buffer.read()) != -1) {

builder.append((char) byteRead);

}

buffer.close();

System.out.println(builder.toString());

解决方案

Try the following code with an added request property:

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.net.URL;

import java.net.URLConnection;

public class SocketConnection

{

public static String getURLSource(String url) throws IOException

{

URL urlObject = new URL(url);

URLConnection urlConnection = urlObject.openConnection();

urlConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");

return toString(urlConnection.getInputStream());

}

private static String toString(InputStream inputStream) throws IOException

{

try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")))

{

String inputLine;

StringBuilder stringBuilder = new StringBuilder();

while ((inputLine = bufferedReader.readLine()) != null)

{

stringBuilder.append(inputLine);

}

return stringBuilder.toString();

}

}

}