在网上查了好长时间,发现httpclient的例子都是讲httpclient 3.X的。httpclient 4.0的例子简直是凤毛麟角。只好自己上官网仔细了查了下文档和官网示例,受益匪浅啊!
由于前段时间对“开心网”的“开心大亨”很感兴趣,但是每隔一段时间就需要看一下最新的价格变化,很是麻烦,由于我就想起用httpclient写个程序,自动查看最新的价格,并和前次的价格做比较,好让我可以选择利润最大的进行投资。
程序用到的包如下,为了附件的瘦身,请大家自己下载啊,附件中是没有这些lib的啊。
commons-collections-3.2.1.jar
commons-configuration-1.6.jar
commons-io-1.4.jar
commons-lang-2.4.jar
commons-logging-1.1.1.jar
httpclient-4.0.jar
httpcore-4.0.1.jar
由于只是例子,而且httpclient4.0的结构变化很大,自己对其理解也不是很深,此程序只是个入门程序,如果有高手在的话,希望大家可以提出宝贵的意见。
程序写的不是很美观,但意在让大家明白如何简单的使用httpclient4.0,所以没有对程序做任何的重构的整理,望大家多多谅解啊。
程序代码如下:
主程序: KaiXin001.java
- /*
- * 2009/12/21
- * Author: Yuan Hongzhi
- */
- import java.io.File;
- import java.io.InputStream;
- import java.util.ArrayList;
- import java.util.List;
- import org.apache.commons.io.FileUtils;
- import org.apache.http.HttpEntity;
- import org.apache.http.HttpResponse;
- import org.apache.http.NameValuePair;
- import org.apache.http.client.entity.UrlEncodedFormEntity;
- import org.apache.http.client.methods.HttpGet;
- import org.apache.http.client.methods.HttpPost;
- import org.apache.http.cookie.Cookie;
- import org.apache.http.entity.BufferedHttpEntity;
- import org.apache.http.impl.client.DefaultHttpClient;
- import org.apache.http.message.BasicNameValuePair;
- import org.apache.http.protocol.HTTP;
- import org.apache.http.util.EntityUtils;
- public class KaiXin001 {
- public InputStream getResourceAsStream(String filename) throws Exception {
- ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- InputStream in = null;
- if (classLoader != null) {
- in = classLoader.getResourceAsStream(filename);
- }
- if (in == null) {
- in = ClassLoader.getSystemResourceAsStream(filename);
- }
- if (in == null) {
- throw new Exception("Can't find resource file: " + filename);
- } else {
- return in;
- }
- }
- public void writeToFile(String file, HttpEntity entity) throws Exception {
- writeToFile(file, EntityUtils.toString(entity));
- }
- public void writeToFile(String file, String data) throws Exception {
- FileUtils.writeStringToFile(new File(file), data, "UTF-8");
- }
- public String getContent(HttpEntity entity) throws Exception {
- if (entity != null) {
- entity = new BufferedHttpEntity(entity);
- long len = entity.getContentLength();
- System.out.println("Length: " + len);
- System.out.println("====================\r\n");
- return EntityUtils.toString(entity, "UTF-8");
- } else {
- System.out.println("entity is null.");
- return null;
- }
- }
- public void setCookie(DefaultHttpClient httpclient, List<Cookie> cookies) {
- if (cookies.isEmpty()) {
- System.out.println("Cookie is empty.");
- return;
- } else {
- for (int i = 0; i < cookies.size(); i++) {
- System.out.println((i + 1) + " - " + cookies.get(i).toString());
- httpclient.getCookieStore().addCookie(cookies.get(i));
- }
- System.out.println();
- }
- }
- // "开心网其它组件URL如下,大家可以添加上自己喜欢的组件URL。"
- // "http://www.kaixin001.com/!slave/index.php", "朋友买卖"
- // "http://www.kaixin001.com/!parking/index.php", "争车位"
- // "http://www.kaixin001.com/!house/index.php?_lgmode=pri", "买房子"
- // "http://www.kaixin001.com/!house/index.php?_lgmode=pri&t=49"
- // "http://www.kaixin001.com/!house/garden/index.php","花园"
- // "http://www.kaixin001.com/!rich/market.php", "超级大亨"
- public String enterComponentContent(String url, String componentName,
- DefaultHttpClient httpclient, List<Cookie> cookies,
- HttpResponse response, HttpEntity entity) throws Exception {
- System.out.println("--- Enter: " + componentName + " ---");
- System.out.println("--- Url: " + url + " ---");
- setCookie(httpclient, cookies);
- HttpGet httpget = new HttpGet(url);
- response = httpclient.execute(httpget);
- entity = response.getEntity();
- return getContent(entity);
- }
- public void showResponseStatus(HttpResponse response) {
- // System.out.println(response.getProtocolVersion());
- // System.out.println(response.getStatusLine().getStatusCode());
- // System.out.println(response.getStatusLine().getReasonPhrase());
- System.out.println(response.getStatusLine().toString());
- System.out.println("-------------------------\r\n");
- }
- public static void main(String[] args) throws Exception {
- KaiXin001 kx = new KaiXin001();
- DefaultHttpClient httpclient = new DefaultHttpClient();
- HttpPost httpost = new HttpPost("http://www.kaixin001.com/login/login.php");
- List<NameValuePair> qparams = new ArrayList<NameValuePair>();
- qparams.add(new BasicNameValuePair("email", "email"));
- qparams.add(new BasicNameValuePair("password", "password"));
- httpost.setEntity(new UrlEncodedFormEntity(qparams, HTTP.UTF_8));
- HttpResponse response = httpclient.execute(httpost);
- kx.showResponseStatus(response);
- // HttpResponse response = httpclient.execute(httpget);
- HttpEntity entity = response.getEntity();
- kx.getContent(entity);
- // Login
- List<Cookie> cookies = httpclient.getCookieStore().getCookies();
- System.out.println("Post logon cookies:");
- kx.setCookie(httpclient, cookies);
- // Redirect to home page
- String homepage = "http://www.kaixin001.com/home/";
- String content = null;
- content = kx.enterComponentContent(homepage, "Home page", httpclient,
- cookies, response, entity);
- // Component
- String componet = "http://www.kaixin001.com/!rich/market.php";
- content = kx.enterComponentContent(componet, "Component", httpclient,
- cookies, response, entity);
- // --------------------------------------------
- kx.writeToFile("c:/kaixin.html", content);
- // When HttpClient instance is no longer needed,
- // shut down the connection manager to ensure
- // immediate deallocation of all system resources
- httpclient.getConnectionManager().shutdown();
- }
- }
/*
* 2009/12/21
* Author: Yuan Hongzhi
*/
import java.io.File;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.cookie.Cookie;
import org.apache.http.entity.BufferedHttpEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HTTP;
import org.apache.http.util.EntityUtils;
public class KaiXin001 {
public InputStream getResourceAsStream(String filename) throws Exception {
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
InputStream in = null;
if (classLoader != null) {
in = classLoader.getResourceAsStream(filename);
}
if (in == null) {
in = ClassLoader.getSystemResourceAsStream(filename);
}
if (in == null) {
throw new Exception("Can't find resource file: " + filename);
} else {
return in;
}
}
public void writeToFile(String file, HttpEntity entity) throws Exception {
writeToFile(file, EntityUtils.toString(entity));
}
public void writeToFile(String file, String data) throws Exception {
FileUtils.writeStringToFile(new File(file), data, "UTF-8");
}
public String getContent(HttpEntity entity) throws Exception {
if (entity != null) {
entity = new BufferedHttpEntity(entity);
long len = entity.getContentLength();
System.out.println("Length: " + len);
System.out.println("====================\r\n");
return EntityUtils.toString(entity, "UTF-8");
} else {
System.out.println("entity is null.");
return null;
}
}
public void setCookie(DefaultHttpClient httpclient, List<Cookie> cookies) {
if (cookies.isEmpty()) {
System.out.println("Cookie is empty.");
return;
} else {
for (int i = 0; i < cookies.size(); i++) {
System.out.println((i + 1) + " - " + cookies.get(i).toString());
httpclient.getCookieStore().addCookie(cookies.get(i));
}
System.out.println();
}
}
// "开心网其它组件URL如下,大家可以添加上自己喜欢的组件URL。"
// "http://www.kaixin001.com/!slave/index.php", "朋友买卖"
// "http://www.kaixin001.com/!parking/index.php", "争车位"
// "http://www.kaixin001.com/!house/index.php?_lgmode=pri", "买房子"
// "http://www.kaixin001.com/!house/index.php?_lgmode=pri&t=49"
// "http://www.kaixin001.com/!house/garden/index.php","花园"
// "http://www.kaixin001.com/!rich/market.php", "超级大亨"
public String enterComponentContent(String url, String componentName,
DefaultHttpClient httpclient, List<Cookie> cookies,
HttpResponse response, HttpEntity entity) throws Exception {
System.out.println("--- Enter: " + componentName + " ---");
System.out.println("--- Url: " + url + " ---");
setCookie(httpclient, cookies);
HttpGet httpget = new HttpGet(url);
response = httpclient.execute(httpget);
entity = response.getEntity();
return getContent(entity);
}
public void showResponseStatus(HttpResponse response) {
// System.out.println(response.getProtocolVersion());
// System.out.println(response.getStatusLine().getStatusCode());
// System.out.println(response.getStatusLine().getReasonPhrase());
System.out.println(response.getStatusLine().toString());
System.out.println("-------------------------\r\n");
}
public static void main(String[] args) throws Exception {
KaiXin001 kx = new KaiXin001();
DefaultHttpClient httpclient = new DefaultHttpClient();
HttpPost httpost = new HttpPost("http://www.kaixin001.com/login/login.php");
List<NameValuePair> qparams = new ArrayList<NameValuePair>();
qparams.add(new BasicNameValuePair("email", "email"));
qparams.add(new BasicNameValuePair("password", "password"));
httpost.setEntity(new UrlEncodedFormEntity(qparams, HTTP.UTF_8));
HttpResponse response = httpclient.execute(httpost);
kx.showResponseStatus(response);
// HttpResponse response = httpclient.execute(httpget);
HttpEntity entity = response.getEntity();
kx.getContent(entity);
// Login
List<Cookie> cookies = httpclient.getCookieStore().getCookies();
System.out.println("Post logon cookies:");
kx.setCookie(httpclient, cookies);
// Redirect to home page
String homepage = "http://www.kaixin001.com/home/";
String content = null;
content = kx.enterComponentContent(homepage, "Home page", httpclient,
cookies, response, entity);
// Component
String componet = "http://www.kaixin001.com/!rich/market.php";
content = kx.enterComponentContent(componet, "Component", httpclient,
cookies, response, entity);
// --------------------------------------------
kx.writeToFile("c:/kaixin.html", content);
// When HttpClient instance is no longer needed,
// shut down the connection manager to ensure
// immediate deallocation of all system resources
httpclient.getConnectionManager().shutdown();
}
}
访问开心大亨组件的程序:KaiXin_Rich.java
- /*
- * 2009/12/21
- * Author: Yuan Hongzhi
- */
- import java.awt.Toolkit;
- import java.text.DecimalFormat;
- import java.util.ArrayList;
- import java.util.List;
- import java.util.regex.Matcher;
- import java.util.regex.Pattern;
- import org.apache.commons.collections.keyvalue.DefaultKeyValue;
- import org.apache.commons.configuration.PropertiesConfiguration;
- import org.apache.http.HttpEntity;
- import org.apache.http.HttpResponse;
- import org.apache.http.NameValuePair;
- import org.apache.http.client.entity.UrlEncodedFormEntity;
- import org.apache.http.client.methods.HttpPost;
- import org.apache.http.cookie.Cookie;
- import org.apache.http.impl.client.DefaultHttpClient;
- import org.apache.http.message.BasicNameValuePair;
- import org.apache.http.protocol.HTTP;
- public class KaiXin_Rich extends KaiXin001 {
- public static List<DefaultKeyValue> curList = new ArrayList<DefaultKeyValue>();
- public static List<DefaultKeyValue> preList = new ArrayList<DefaultKeyValue>();
- public void copyList(List<DefaultKeyValue> src, List<DefaultKeyValue> des) {
- for (DefaultKeyValue dkv : src) {
- DefaultKeyValue kv = new DefaultKeyValue(dkv.getKey(), dkv.getValue());
- des.add(kv);
- }
- }
- public Double subNumber(Object string) {
- String rex = "([\\d\\.]*)";
- Pattern pattern = Pattern.compile(rex);
- Matcher match = pattern.matcher(string.toString());
- if (match.find()) {
- return Double.valueOf(match.group());
- } else
- return -1.0;
- }
- public String extractNoneAscii(Object string) {
- String rex = "([^\\x00-\\xff]+)";
- Pattern pattern = Pattern.compile(rex);
- Matcher match = pattern.matcher(string.toString());
- if (match.find()) {
- return match.group();
- } else
- return "";
- }
- public static void main(String[] args) throws Exception {
- KaiXin_Rich kx = new KaiXin_Rich();
- PropertiesConfiguration config = new PropertiesConfiguration();
- config.load(kx.getResourceAsStream("properties/kaixin001.properties"),
- "UTF-8");
- DefaultHttpClient httpclient = new DefaultHttpClient();
- HttpPost httpost = new HttpPost("http://www.kaixin001.com/login/login.php");
- List<NameValuePair> qparams = new ArrayList<NameValuePair>();
- qparams.add(new BasicNameValuePair("email", config.getString("email")));
- qparams
- .add(new BasicNameValuePair("password", config.getString("password")));
- httpost.setEntity(new UrlEncodedFormEntity(qparams, HTTP.UTF_8));
- HttpResponse response = httpclient.execute(httpost);
- kx.showResponseStatus(response);
- // HttpResponse response = httpclient.execute(httpget);
- HttpEntity entity = response.getEntity();
- kx.getContent(entity);
- // Login
- List<Cookie> cookies = httpclient.getCookieStore().getCookies();
- System.out.println("Post logon cookies:");
- kx.setCookie(httpclient, cookies);
- // Redirect to home page
- String homepage = "http://www.kaixin001.com/home/";
- String content = null;
- content = kx.enterComponentContent(homepage, "Home page", httpclient,
- cookies, response, entity);
- int loopTime = config.getInt("times");
- long sleepTime = config.getLong("period") * 60;
- for (int j = 0; j < loopTime; j++) {
- // Component
- String componet = config.getString("url");
- content = kx.enterComponentContent(componet, "Component", httpclient,
- cookies, response, entity);
- int beginIndex = content.indexOf(config.getString("startRex"));
- int endIndex = content.lastIndexOf(config.getString("endRex"));
- content = content.substring(beginIndex, endIndex);
- // --------------------------------------------
- String rex = config.getString("displayKeyRex");
- Pattern pattern = Pattern.compile(rex, Pattern.MULTILINE);
- Matcher match = pattern.matcher(content);
- List<String> keys = new ArrayList<String>();
- String matched = null;
- while (match.find()) {
- matched = match.group();
- keys.add(matched);
- }
- rex = config.getString("displayValueRex");
- pattern = Pattern.compile(rex, Pattern.MULTILINE);
- match = pattern.matcher(content);
- List<String> values = new ArrayList<String>();
- while (match.find()) {
- matched = match.group();
- values.add(matched);
- }
- DefaultKeyValue kv = null;
- for (int i = 0; i < keys.size(); i++) {
- kv = new DefaultKeyValue(keys.get(i), values.get(i));
- curList.add(kv);
- }
- String rtnString = "";
- String eachLine = null;
- for (DefaultKeyValue dkv : curList) {
- eachLine = dkv.getKey() + "\t:" + dkv.getValue();
- rtnString += eachLine;
- rtnString += "\r\n";
- System.out.println(eachLine);
- }
- System.out.println("\r\n===== Changed =====\r\n");
- String allPriceString = "===== Changed =====\r\n\r\n";
- StringBuffer price = null;
- double changed = 0;
- boolean beepFlag = false;
- for (int i = 0; i < curList.size() && curList.size() == preList.size(); i++) {
- double curMoney = kx.subNumber(curList.get(i).getValue());
- double preMoney = kx.subNumber(preList.get(i).getValue());
- if ((curMoney - preMoney) > 1E-5) {
- changed = curMoney - preMoney;
- DecimalFormat format = new DecimalFormat("###.##");
- String number = format.format(changed);
- price = new StringBuffer();
- price.append(curList.get(i).getKey());
- price.append("\t: ");
- price.append(preList.get(i).getValue());
- price.append(" -> ");
- price.append(curList.get(i).getValue());
- price.append("\t");
- price.append(" Changed: ");
- price.append(number);
- price.append(kx.extractNoneAscii(curList.get(i).getValue()));
- System.out.println(price);
- allPriceString = allPriceString + price.toString() + "\r\n";
- beepFlag = true;
- }
- }
- if (beepFlag) {
- Toolkit.getDefaultToolkit().beep();
- }
- preList.clear();
- // System.out.println(content);
- String dateTime = DateUtils
- .getSystemDateTime(DateTimePattern.DATE_TIME_LONG_UNFORMAT);
- kx.writeToFile("c:/kaixin001/price_" + dateTime + ".txt", rtnString
- + "\r\n" + allPriceString);
- // kx.writeToFile("c:/kaixin.html", content);
- kx.copyList(curList, preList);
- curList.clear();
- Thread.sleep(sleepTime * 1000);
- }
- // When HttpClient instance is no longer needed,
- // shut down the connection manager to ensure
- // immediate deallocation of all system resources
- httpclient.getConnectionManager().shutdown();
- }
- }
/*
* 2009/12/21
* Author: Yuan Hongzhi
*/
import java.awt.Toolkit;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.collections.keyvalue.DefaultKeyValue;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.cookie.Cookie;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HTTP;
public class KaiXin_Rich extends KaiXin001 {
public static List<DefaultKeyValue> curList = new ArrayList<DefaultKeyValue>();
public static List<DefaultKeyValue> preList = new ArrayList<DefaultKeyValue>();
public void copyList(List<DefaultKeyValue> src, List<DefaultKeyValue> des) {
for (DefaultKeyValue dkv : src) {
DefaultKeyValue kv = new DefaultKeyValue(dkv.getKey(), dkv.getValue());
des.add(kv);
}
}
public Double subNumber(Object string) {
String rex = "([\\d\\.]*)";
Pattern pattern = Pattern.compile(rex);
Matcher match = pattern.matcher(string.toString());
if (match.find()) {
return Double.valueOf(match.group());
} else
return -1.0;
}
public String extractNoneAscii(Object string) {
String rex = "([^\\x00-\\xff]+)";
Pattern pattern = Pattern.compile(rex);
Matcher match = pattern.matcher(string.toString());
if (match.find()) {
return match.group();
} else
return "";
}
public static void main(String[] args) throws Exception {
KaiXin_Rich kx = new KaiXin_Rich();
PropertiesConfiguration config = new PropertiesConfiguration();
config.load(kx.getResourceAsStream("properties/kaixin001.properties"),
"UTF-8");
DefaultHttpClient httpclient = new DefaultHttpClient();
HttpPost httpost = new HttpPost("http://www.kaixin001.com/login/login.php");
List<NameValuePair> qparams = new ArrayList<NameValuePair>();
qparams.add(new BasicNameValuePair("email", config.getString("email")));
qparams
.add(new BasicNameValuePair("password", config.getString("password")));
httpost.setEntity(new UrlEncodedFormEntity(qparams, HTTP.UTF_8));
HttpResponse response = httpclient.execute(httpost);
kx.showResponseStatus(response);
// HttpResponse response = httpclient.execute(httpget);
HttpEntity entity = response.getEntity();
kx.getContent(entity);
// Login
List<Cookie> cookies = httpclient.getCookieStore().getCookies();
System.out.println("Post logon cookies:");
kx.setCookie(httpclient, cookies);
// Redirect to home page
String homepage = "http://www.kaixin001.com/home/";
String content = null;
content = kx.enterComponentContent(homepage, "Home page", httpclient,
cookies, response, entity);
int loopTime = config.getInt("times");
long sleepTime = config.getLong("period") * 60;
for (int j = 0; j < loopTime; j++) {
// Component
String componet = config.getString("url");
content = kx.enterComponentContent(componet, "Component", httpclient,
cookies, response, entity);
int beginIndex = content.indexOf(config.getString("startRex"));
int endIndex = content.lastIndexOf(config.getString("endRex"));
content = content.substring(beginIndex, endIndex);
// --------------------------------------------
String rex = config.getString("displayKeyRex");
Pattern pattern = Pattern.compile(rex, Pattern.MULTILINE);
Matcher match = pattern.matcher(content);
List<String> keys = new ArrayList<String>();
String matched = null;
while (match.find()) {
matched = match.group();
keys.add(matched);
}
rex = config.getString("displayValueRex");
pattern = Pattern.compile(rex, Pattern.MULTILINE);
match = pattern.matcher(content);
List<String> values = new ArrayList<String>();
while (match.find()) {
matched = match.group();
values.add(matched);
}
DefaultKeyValue kv = null;
for (int i = 0; i < keys.size(); i++) {
kv = new DefaultKeyValue(keys.get(i), values.get(i));
curList.add(kv);
}
String rtnString = "";
String eachLine = null;
for (DefaultKeyValue dkv : curList) {
eachLine = dkv.getKey() + "\t:" + dkv.getValue();
rtnString += eachLine;
rtnString += "\r\n";
System.out.println(eachLine);
}
System.out.println("\r\n===== Changed =====\r\n");
String allPriceString = "===== Changed =====\r\n\r\n";
StringBuffer price = null;
double changed = 0;
boolean beepFlag = false;
for (int i = 0; i < curList.size() && curList.size() == preList.size(); i++) {
double curMoney = kx.subNumber(curList.get(i).getValue());
double preMoney = kx.subNumber(preList.get(i).getValue());
if ((curMoney - preMoney) > 1E-5) {
changed = curMoney - preMoney;
DecimalFormat format = new DecimalFormat("###.##");
String number = format.format(changed);
price = new StringBuffer();
price.append(curList.get(i).getKey());
price.append("\t: ");
price.append(preList.get(i).getValue());
price.append(" -> ");
price.append(curList.get(i).getValue());
price.append("\t");
price.append(" Changed: ");
price.append(number);
price.append(kx.extractNoneAscii(curList.get(i).getValue()));
System.out.println(price);
allPriceString = allPriceString + price.toString() + "\r\n";
beepFlag = true;
}
}
if (beepFlag) {
Toolkit.getDefaultToolkit().beep();
}
preList.clear();
// System.out.println(content);
String dateTime = DateUtils
.getSystemDateTime(DateTimePattern.DATE_TIME_LONG_UNFORMAT);
kx.writeToFile("c:/kaixin001/price_" + dateTime + ".txt", rtnString
+ "\r\n" + allPriceString);
// kx.writeToFile("c:/kaixin.html", content);
kx.copyList(curList, preList);
curList.clear();
Thread.sleep(sleepTime * 1000);
}
// When HttpClient instance is no longer needed,
// shut down the connection manager to ensure
// immediate deallocation of all system resources
httpclient.getConnectionManager().shutdown();
}
}
个人签名
-------------------------------------