抓取数据

package net.wocai.tools.spider;


import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;


import net.wocai.domain.Brand;
import net.wocai.domain.Category;
import net.wocai.domain.City;
import net.wocai.domain.Company;
import net.wocai.domain.Job;
import net.wocai.domain.Product;
import net.wocai.domain.ProductPrice;
import net.wocai.util.DateUtil;


import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.*;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFCellStyle;
import org.apache.poi.hssf.usermodel.HSSFFont;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.hssf.util.HSSFColor;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


public class fCopyOf58 {


private final static HttpClient client = new DefaultHttpClient();
private static String domain = "http://hz.58.com/";// 必须以http://**.gldjc.com/格式定义
// ;以http开头,以/结尾


// public static List<ProductPrice> productprice = new
// ArrayList<ProductPrice>();


public static void main(String[] args) throws Exception {
// Map<String, String> map = new HashMap<String, String>();
// map.put("http://sx.gldjc.com/", "610000");
// String string = get_html(domain+"/it");
// string = get_html(domain);
// List<Link> job=getJob(domain+"/it");
// get_phone("http://image.58.com/showphone.aspx?t=v55&v=736EE56F482E3F5A6D4A5694995914F70");
// System.out.println("");
// String
// tel=get_phone("http://image.58.com/showphone.aspx?t=v55&v=736EE56F482E3F5A6D4A5694995914F70");
// String valCode = new OCR().recognizeText(new File(tel));
// System.out.println(valCode);
getJobByIndustry(domain + "/xueshengjianzhi/");
System.out.println("/");
// get_html("http://hz.1010jz.com");
// getJobByIndustry(domain+"/it/");
// System.out.println(job.size());
// System.out.println(job.get(0).getLink());
// System.out.println(job.size());
// for(int i=0;i<job.size();i++){
// String s;
// String l=job.get(i).getLink();
// String n=job.get(i).getCategory();
// String[] d=n.split("/");
// if(d.length==2){
// s=d[0]+","+d[1];
// }else{
// s=n;
// }
// String url=domain+l;
// getJobByIndustry(url,s);
// }


}


static void getJobByIndustry(String s) throws Exception {
for (int i = 1; i < 3; i++) {
String url = s + "pn" + i;
String jobinfo = get_html(url);
getJobInfo(jobinfo);
}


}


private static void getJobInfo(String html) throws Exception {
// Elements links = getElements(html, "li");
Elements links = getElementsByclassname(html, "t");
List<Job> list = new ArrayList<Job>();
for (int i = 0; i < links.size(); i++) {
Job job = new Job();
Element element = links.get(i);
// Elements jobinfo=link.getElementsByAttribute("a");
// System.out.println(element.attr("href"));
// Link al = new Link();
// al.setLink(element.attr("href"));
String href = element.attr("href");
String info = get_html(href);
Document document = Jsoup.parse(info);
// String title=document.getElementsByTag("h2").first().text();
Element element2 = document.getElementsByClass("leftbar").first();
if (element2 == null) {
System.out.println(href);
continue;
}
String title = element2.getElementsByTag("h1").first().text();
job.setTitle(title);
String cname = element2.getElementsByTag("a").get(0).text();
job.setCname(cname);
// job.setPlace(place);
String d2 = element2.text();
String[] d21 = d2.split("招聘人数：");
String[] d211 = d21[1].split("有效期：");
job.setPnumber(d211[0]);
String[] d212 = d211[1].split("薪资水平：");
job.setTime(d212[0]);
String[] d213 = d212[1].split("兼职时间：");
job.setSalary(d213[0]);
String[] d214 = d213[1].split("工作区域：");
job.setWorktime(d214[0]);
String[] d215 = d214[1].split("立即申请");
job.setPlace(d215[0]);
Element tx = document.getElementById("lianxi");
String t = tx.text();
String[] t1 = t.split("联系人：");
String[] t2 = t1[0].split("联系电话：");
String pname = t2[0];
String[] t3 = t2[1].split("在线沟通：");
String telephone = t3[0];
String address;
if(t3.length==1){
address = "未填写";
}else{
String[] t4 = t3[1].split("联系地址：");
address = t4[1];
}
job.setAddress(address);
Elements e = tx.getElementsByTag("img");
if (e.size() == 2 || e.size() == 3) {
String id = document.getElementById("pagenum").val();
String src = e.get(0).attr("src");
String[] d = id.split("_");
if (d.length == 2) {
String s = src + d[0];
telephone = get_phone(s);
String v1 = new OCR().recognizeText(new File(telephone));
String s2 = src + d[1];
telephone = get_phone(s2);
String v2 = new OCR().recognizeText(new File(telephone));
String valCode = v1 + v2;
telephone = valCode;
} else {
src = src + id;
telephone = get_phone(src);
String valCode = new OCR()
.recognizeText(new File(telephone));
telephone = valCode;
}
}

job.setPname(pname);
job.setTelephone(telephone);

list.add(job);
}
exportJobinfo(list);// 生成文件
}


private static void exportJobinfo(List<Job> list) {
String name = System.currentTimeMillis() + ".xls";
HSSFWorkbook workbook = exportJob(list);
try {
FileOutputStream outputStream = new FileOutputStream("e://58//"
+ name);
workbook.write(outputStream);
outputStream.flush();
outputStream.close();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}


}


private static HSSFWorkbook exportJob(List<Job> list) {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss");
// 声明一个工作薄
HSSFWorkbook workbook = new HSSFWorkbook();
// 生成一个表格
HSSFSheet sheet = workbook.createSheet("工作信息");
sheet.setColumnWidth(0, 18 * 128);
sheet.setColumnWidth(1, 18 * 512);
sheet.setColumnWidth(2, 18 * 512);
sheet.setColumnWidth(3, 18 * 512);
sheet.setColumnWidth(4, 18 * 256);
sheet.setColumnWidth(5, 18 * 256);
sheet.setColumnWidth(6, 18 * 256);
sheet.setColumnWidth(7, 18 * 256);
sheet.setColumnWidth(8, 18 * 256);
sheet.setColumnWidth(9, 18 * 256);
// 生成一个样式
HSSFCellStyle style = workbook.createCellStyle();
// 设置这些样式
style.setFillForegroundColor(HSSFColor.SKY_BLUE.index);
style.setFillPattern(HSSFCellStyle.SOLID_FOREGROUND);
style.setBorderBottom(HSSFCellStyle.BORDER_THIN);
style.setBorderLeft(HSSFCellStyle.BORDER_THIN);
style.setBorderRight(HSSFCellStyle.BORDER_THIN);
style.setBorderTop(HSSFCellStyle.BORDER_THIN);
style.setAlignment(HSSFCellStyle.ALIGN_CENTER);
// 生成一个字体
HSSFFont font = workbook.createFont();
font.setColor(HSSFColor.VIOLET.index);
font.setFontHeightInPoints((short) 12);
font.setBoldweight(HSSFFont.BOLDWEIGHT_BOLD);
// 把字体应用到当前的样式
// style.setFont(font);


HSSFCellStyle styledata = workbook.createCellStyle();
styledata.setAlignment(HSSFCellStyle.ALIGN_CENTER);


HSSFRow row = sheet.createRow(0);
// 1列
HSSFCell cell = row.createCell(0);
cell.setCellStyle(style);
cell.setCellValue("工作名称");
//
cell = row.createCell(1);
cell.setCellStyle(style);
cell.setCellValue("公司名称");
// 3列
cell = row.createCell(2);
cell.setCellStyle(style);
cell.setCellValue("招聘人数");


cell = row.createCell(3);
cell.setCellStyle(style);
cell.setCellValue("有效期");


cell = row.createCell(4);
cell.setCellStyle(style);
cell.setCellValue("薪资水平");


cell = row.createCell(5);
cell.setCellStyle(style);
cell.setCellValue("兼职时间");
//
cell = row.createCell(6);
cell.setCellStyle(style);
cell.setCellValue("工作区域");


cell = row.createCell(7);
cell.setCellStyle(style);
cell.setCellValue("联系人");


cell = row.createCell(8);
cell.setCellStyle(style);
cell.setCellValue("联系电话");


cell = row.createCell(9);
cell.setCellStyle(style);
cell.setCellValue("联系地址");


if (list != null && list.size() > 0) {
for (int i = 0; i < list.size(); i++) {
Job job = list.get(i);
HSSFRow content = sheet.createRow(i + 1);
cell = content.createCell(0);
cell.setCellValue(job.getTitle());
cell = content.createCell(1);
cell.setCellValue(job.getCname());
cell = content.createCell(2);
cell.setCellValue(job.getPnumber());
cell = content.createCell(3);
cell.setCellValue(job.getTime());
cell = content.createCell(4);
cell.setCellValue(job.getSalary());
cell = content.createCell(5);
cell.setCellValue(job.getWorktime());
cell = content.createCell(6);
cell.setCellValue(job.getPlace());
cell = content.createCell(7);
cell.setCellValue(job.getPname());
cell = content.createCell(8);
cell.setCellValue(job.getTelephone());
cell = content.createCell(9);
cell.setCellValue(job.getAddress());
}
}


return workbook;
}


private static String get_phone(String src) throws ClientProtocolException,
IOException {
String filename = System.currentTimeMillis() + ".png";
URL url = new URL(src);
URLConnection con = url.openConnection();
InputStream is = con.getInputStream();
byte[] bs = new byte[1024];
int len;
OutputStream os = new FileOutputStream("e://58//" + filename);
while ((len = is.read(bs)) != -1) {
os.write(bs, 0, len);
}
filename = "e://58//" + filename;
os.close();
is.close();
return filename;
}


private static List<Link> getJob(String domain2) throws IOException {
Document document = Jsoup.parse(get_html(domain2));
Element element = document.getElementsByClass("listclass").first();
Elements a_elements = element.getElementsByTag("a");
List<Link> list = new ArrayList<Link>();
for (int i = 1; i < 18; i++) {
Link al = new Link();
al.setLink(a_elements.get(i).attr("href"));
al.setCategory(a_elements.get(i).text());
list.add(al);
}
return list;
}


/**
* 
* <一句话描述功能>根据一个二级分类列表地址循环抓取产品详细内容,每个连接延迟5秒执行</一句话描述功能>
* 
* @param link
*/
// http://bj.gldjc.com/search/product/list_291541_110___110000________40__.html
// http://bj.gldjc.com/search/product/list_291541_110___110000__________.html
static void getproductpage(List<Link> link, List<ProductPrice> productprice) {
for (Link l : link) {
System.out.println(DateUtil.getCurrentDateTime());
String alink = l.getLink();
String category = l.getCategory();
try {
String product = get_html(domain + alink);
System.out.println("二级类目:" + domain + alink);
int pagesize = getProdcutPageInfo(product);// 获取分页总数


getProductInfo(product, category, productprice);
String pageLink = getProdcutPageLink(product);
String pagetop = "", pageend = "";
if (StringUtils.isNotEmpty(pageLink)) {
String[] pages = pageLink.split("_______");
pagetop = pages[0] + "_______";
pageend = StringUtils.replace(pages[1], "2_", "");
}
for (int i = 2; i <= pagesize; i++) {
String pagelink = pagetop + i + "_" + pageend;
String pageproduct = get_html(domain + pagelink);
System.out.println("******分页:" + domain + pagelink);
getProductInfo(pageproduct, category, productprice);
// try {
// Thread.sleep(1000);
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
}
} catch (IOException e1) {
e1.printStackTrace();
System.out.println("抓取产品列表异常");
}
// try {
// Thread.sleep(25000);
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
}
}


static void getproductpage2(List<Link> link, List<ProductPrice> productprice) {
for (Link l : link) {
System.out.println(DateUtil.getCurrentDateTime());
String alink = l.getLink();
String category = l.getCategory();
try {
String product = get_html(domain + alink);
System.out.println("二级类目:" + domain + alink);
int pagesize = getProdcutPageInfo(product);// 获取分页总数
int total = pagesize * 20;
int newpagesize = (total / 2800) + 1;
// getProductInfo(product, category, productprice);
String pageLink = getProdcutPageLink(product);
String pagetop = "", pageend = "";
if (StringUtils.isNotEmpty(pageLink)) {
String[] pages = pageLink.split("_______");
pagetop = pages[0] + "_______";
pageend = StringUtils.replace(pages[1], "2_20", "");
for (int i = 1; i <= newpagesize; i++) {
String pagelink = pagetop + i + "_2800" + pageend;
String pageproduct = get_html(domain + pagelink);
System.out.println("******分页:" + domain + pagelink);
getProductInfo(pageproduct, category, productprice);
// try {
// Thread.sleep(1000);
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
}
} else {
getProductInfo(product, category, productprice);
}


} catch (IOException e1) {
e1.printStackTrace();
System.out.println("抓取产品列表异常");
}
// try {
// Thread.sleep(25000);
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
}
}


// http://bj.gldjc.com/search/product/list_291541_110___110000________40__.html
// http://bj.gldjc.com/search/product/list_291541_110___110000__________.html
static void getproductforsettotal(List<Link> link,
List<ProductPrice> productprice) {
for (Link l : link) {
System.out.println(DateUtil.getCurrentDateTime());
String alink = l.getLink();
String category = l.getCategory();
try {
String url = domain + alink;
String product = get_html(url);
int pagesize = getProdcutPageInfo(product);// 获取分页总数
int total = pagesize * 20;
String pageurl = StringUtils.replace(url, "__.html", total
+ "__.html");
System.out.println("二级类目:" + pageurl);
String pageproduct = get_html(pageurl);
getProductInfo(pageproduct, category, productprice);


} catch (IOException e1) {
e1.printStackTrace();
System.out.println("抓取产品列表异常");
}
// try {
// Thread.sleep(25000);
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
}
}


static List<Link> getProductClassPageForJS(List<Link> link,
Map<String, String> map) throws IOException {
List<Link> list = new ArrayList<Link>();
String s = "/search/product_search_bmts_nav?search[first_type_id]=25&amp;search[location_area_code]=340000";
for (Link l : link) {
String alink = l.getLink();
System.out.println(domain + alink);
String cityid = StringUtils.replace(alink,
"/search/product/list__", "");
cityid = StringUtils.replace(cityid, "_____________.html", "");
String classurl = "/search/product_search_bmts_nav?search[first_type_id]="
+ cityid
+ "&amp;search[location_area_code]="
+ map.get(domain);
String product = get_html(domain + classurl);
Document document = Jsoup.parse(product);
Elements elements = document.getElementsByTag("a");
for (Element link_a : elements) {
Link al = new Link();
al.setLink(link_a.attr("href"));
// System.out.println("----"+al.getLink());
al.setCategory(link_a.attr("title"));
// System.out.println("----"+al.getCategory());
list.add(al);
}
// for (Element e : elements) {
// System.out.println(e.text());
// String a = e.html();
// Elements links_as = getElements(a, "a");
// for (Element link_a : links_as) {
// Link al = new Link();
// al.setLink(link_a.attr("href"));
// al.setCategory(link_a.attr("title"));
// list.add(al);
// }
// }
}
return list;
}


static List<Link> getProductClassPage(List<Link> link) throws IOException {
List<Link> list = new ArrayList<Link>();
for (Link l : link) {
String alink = l.getLink();
String product = get_html(domain + alink);
Document document = Jsoup.parse(product);
Elements elements = document.getElementsByClass("subnav-left");
for (Element e : elements) {
System.out.println(e.text());
String a = e.html();
Elements links_as = getElements(a, "a");
for (Element link_a : links_as) {
Link al = new Link();
al.setLink(link_a.attr("href"));
al.setCategory(link_a.attr("title"));
list.add(al);
}
}
}
return list;
}


/**
* 
* <一句话描述功能>根据供应商产品html页面处理供应商产品信息</一句话描述功能>
* 
* @param html
* @return
*/
static List<Link> getProductInfo(String html, String category,
List<ProductPrice> productprice) throws IOException {
Elements links = getElements(html, "tr");
List<Link> list = new ArrayList<Link>();
for (Element link : links) {
Elements productinfo = link.getElementsByTag("td");
int i = 0;
ProductPrice priceobj = new ProductPrice();
Product product = new Product();
Brand brand = new Brand();
Company com = new Company();
City city = new City();
Category categoryobj = new Category();
categoryobj.setTitle(category);
product.setCategory(categoryobj);
for (Element p : productinfo) {
// System.out.println(p.text());
switch (i) {
case 0:// 产品名称
// System.out.print(p.text() + " | ");
product.setName(p.text());
break;
case 1:// 规格型号
// System.out.print(p.text() + " | ");
product.setFormat(p.text());
break;
case 2:// 工程价


break;
case 3:// 市场价
Elements imgs = getElements(p.html(), "img");
String src = imgs.get(0).attr("src");
// String price = src;
String price = get_price(src);
// System.out.print(price + " | ");
priceobj.setPrice(price);
break;
case 4:// 单位
// System.out.print(p.text() + " | ");
product.setMeasuring(p.text());
break;
case 5:// 品牌
// System.out.print(p.text() + " | ");


brand.setName(p.text());
product.setBrand(brand);
break;
case 6:// 地区
// System.out.print(p.text() + " | ");
city.setName(p.text());
priceobj.setCity(city);
break;
case 7:// 供应商
String company = p.text();
company = StringUtils.replace(company, "【合作供应商】", "");
company = company.trim();
// System.out.print(company + " | ");
com.setName(company);
product.setCompany(com);
break;


case 8:// 报价日期
// System.out.print(p.text() + " | ");
priceobj.setCreated(DateUtil.StringToDate(p.text() + "-1",
"yyyy-MM-dd"));
break;


}


i++;
}
priceobj.setProduct(product);
productprice.add(priceobj);
// System.out.println();
// System.out
// .println("--------------------------------------------------------------");
// String text = link.text();
// System.out.println(text);
}
return list;
}


/**
* 抓取网页
* 
* @param url
* @throws IOException
*/
static void get(String url) throws IOException {
HttpGet get = new HttpGet(url);
HttpResponse response = client.execute(get);
System.out.println(response.getStatusLine());


HttpEntity entity = response.getEntity();
String html = EntityUtils.toString(response.getEntity());


System.out.println(html);


// dump(entity);
}


/**
* 
* <一句话描述功能>根据url下载图片取头文件信息中的文件名称 分析报价</一句话描述功能>
* 
* @param url
* @return
* @throws IOException
*/
static String get_price(String url) throws IOException {
HttpGet get = new HttpGet(url);
HttpResponse response = client.execute(get);
String price = "0";
Header[] headers = response.getAllHeaders();


for (int i = 0; i < headers.length; i++) {
if ("Content-Encoding".equalsIgnoreCase(headers[i].getName())) {
System.out.println(headers[i].getValue());
}


if ("Content-Disposition".equals(headers[i].getName())) {
price = new String(headers[i].getValue().getBytes("ISO8859_1"),
"utf-8");
if (price.startsWith("inline; filename=")) {
price = StringUtils.replace(price, "inline; filename=\"",
"");
price = StringUtils.replace(price, ".png\"", "");
}
break;
}


}
EntityUtils.toString(response.getEntity());
return price;


}


public static String inputStream2String(InputStream is) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
int i = -1;
System.out.println(is.read());
while ((i = is.read()) != -1) {
baos.write(i);
}
return baos.toString();
}


/**
* 
* <一句话描述功能>返回页面html文本</一句话描述功能>
* 
* @param url
* @return
* @throws IOException
*/
static String get_html(String url) throws IOException {
try {
HttpGet get = new HttpGet(url);
HttpResponse response = client.execute(get);
String con = EntityUtils.toString(response.getEntity(), "gb2312");
// BufferedReader br = new BufferedReader(new
// InputStreamReader(response.getEntity().getContent(),"gb2312"));
// String con = IOUtils.toString(br);
// System.out.println(con);
return con;
} catch (Exception e) {
return get_html(url);
}
}


static List<Link> getClassMoreLinkByIndexHtml(String domain)
throws IOException {
String con = get_html(domain);
Element e = getElementsByclassname(con, "screen-2").get(0)
.getElementsByClass("left").get(0);
Elements es = e.select("span").select("a");
List<Link> links = new ArrayList<Link>();
for (Element e1 : es) {
Link link = new Link();
link.setLink(e1.attr("href"));
links.add(link);
}


return links;
}


/**
* 
* <一句话描述功能>返回一个一级分类下的 所有二级分类详细产品的链接地址集合</一句话描述功能>
* 
* @param html
* @return
*/
static List<Link> getClassURL(String html) {
Elements links = getElements(html, "dt");
List<Link> list = new ArrayList<Link>();
for (Element link : links) {
String a = link.html();
Elements links_as = getElements(a, "a");
for (Element link_a : links_as) {
Link l = new Link();
l.setLink(link_a.attr("href"));
list.add(l);
}
}
return list;
}


/**
* 
* <一句话描述功能>取总页数</一句话描述功能>
* 
* @param html
* @return
*/
static int getProdcutPageInfo(String html) {
int pagesize = 1;
Document document = Jsoup.parse(html);
Elements elements = document.getElementsByClass("go");
for (Element e : elements) {
String text = e.text();
text = StringUtils.replace(text, "共", "");
text = StringUtils.replace(text, "页 到", "");
pagesize = Integer.parseInt(text);
}
return pagesize;
}


/**
* 
* <一句话描述功能>取分页链接</一句话描述功能>
* 
* @param html
* @return
*/


static String getProdcutPageLink(String html) {
Document document = Jsoup.parse(html);
Elements elements = document.getElementsByClass("flickr_pagination");
String pageLink = "";
for (Element e : elements) {
String a = e.html();
Elements links_as = getElements(a, "a");
for (Element link_a : links_as) {
pageLink = link_a.attr("href");
break;
}
}
return pageLink;
}


/**
* 
* <一句话描述功能>指定一段html,根据标签提取标签内的内容</一句话描述功能>
* 
* @param html
*            html代码内容
* @param tagName
*            html标签名
* @return
*/
static Elements getElements(String html, String tagName) {
Document document_a = Jsoup.parse(html);
return document_a.getElementsByTag(tagName);
}


/**
* 
* <一句话描述功能>指定一段html,根据ClassName提取标签内的内容</一句话描述功能>
* 
* @param html
* @param classname
* @return
*/
static Elements getElementsByclassname(String html, String classname) {
Document document_a = Jsoup.parse(html);
return document_a.getElementsByClass(classname);
}


/**
* 执行登录过程
* 
* @param user
* @param pwd
* @param debug
* @throws IOException
*/
static void login(String user, String pwd, boolean debug)
throws IOException {
CookieStore cookieStore = new BasicCookieStore();
// 创建一个本地上下文信息
HttpContext localContext = new BasicHttpContext();
// 在本地上下问中绑定一个本地存储
localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);


HttpPost post = new HttpPost("http://sso.glodon.com/sessions/login");
post.setHeader(
"User-Agent",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3");


// 登录表单的信息
List<NameValuePair> qparams = new ArrayList<NameValuePair>();
qparams.add(new BasicNameValuePair("utf8", "&#x2713"));


qparams.add(new BasicNameValuePair("loginname", user));
qparams.add(new BasicNameValuePair("password", pwd));
qparams.add(new BasicNameValuePair("lt", ""));
qparams.add(new BasicNameValuePair("service",
"http://www.gldjc.com/user/auth/cas/callback?url=http%3A%2F%2Fwww.gldjc.com%2F"));
qparams.add(new BasicNameValuePair("commit", "登录"));


UrlEncodedFormEntity params = new UrlEncodedFormEntity(qparams, "UTF-8");
post.setEntity(params);


// Execute the request
HttpResponse response = client.execute(post);


if (debug) {
// Examine the response status
System.out.println(response.getStatusLine());
// Get hold of the response entity
HttpEntity entity = response.getEntity();
System.out.println("server return code:"
+ response.getStatusLine().getStatusCode());
dump(entity);
}
// 获取response中location的值
Header header = (Header) response.getFirstHeader("Location");
get(header.getValue());// 广联达sso登录,回调授权
System.out.println(header.getValue());
// System.out.println("Post logon cookies:");
//
// List<Cookie> cookies =cookieStore.getCookies();
// if (cookies.isEmpty()) {
// System.out.println("None");
// } else {
// for (int i = 0; i < cookies.size(); i++) {
// System.out.println("- " + cookies.get(i).toString());
// }
// }
// //获取消息头的信息
// Header[] headers = response.getAllHeaders();
// for (int i = 0; i<headers.length; i++) {
// System.out.println(headers[i]);
// }


// get("http://www.gldjc.com");


// http://www.gldjc.com/user/auth/cas/callback?url=http%3A%2F%2Fwww.gldjc.com%2F&ticket=ST-4ce263ae06717a4ad64ddb39aad95
post.abort();


}


static void newlogin(String user, String pwd, boolean debug)
throws IOException {
CookieStore cookieStore = new BasicCookieStore();
// 创建一个本地上下文信息
HttpContext localContext = new BasicHttpContext();
// 在本地上下问中绑定一个本地存储
localContext.setAttribute(ClientContext.COOKIE_STORE, cookieStore);


HttpPost post = new HttpPost(
"https://account.glodon.com/j_spring_security_check");
post.setHeader(
"User-Agent",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3");


// 登录表单的信息
List<NameValuePair> qparams = new ArrayList<NameValuePair>();
qparams.add(new BasicNameValuePair("utf8", "&#x2713"));
qparams.add(new BasicNameValuePair("service_key",
"E6xtqZdaM3D0no2jG9Q6p5yisfh4WmJJ"));
qparams.add(new BasicNameValuePair("j_username", user));
qparams.add(new BasicNameValuePair("j_password", pwd));
qparams.add(new BasicNameValuePair("service",
"http://www.gldjc.com/user/cas/yun_callback?url=http%3A%2F%2Fwww.gldjc.com"));
qparams.add(new BasicNameValuePair("commit", "登录"));


UrlEncodedFormEntity params = new UrlEncodedFormEntity(qparams, "UTF-8");
post.setEntity(params);


// Execute the request
HttpResponse response = client.execute(post);


if (debug) {
// Examine the response status
System.out.println(response.getStatusLine());
// Get hold of the response entity
HttpEntity entity = response.getEntity();
System.out.println("server return code:"
+ response.getStatusLine().getStatusCode());
dump(entity);
}
// 获取response中location的值
Header header = (Header) response.getFirstHeader("Location");
get(header.getValue());// 广联达sso登录,回调授权
System.out.println(header.getValue());
// System.out.println("Post logon cookies:");
//
// List<Cookie> cookies =cookieStore.getCookies();
// if (cookies.isEmpty()) {
// System.out.println("None");
// } else {
// for (int i = 0; i < cookies.size(); i++) {
// System.out.println("- " + cookies.get(i).toString());
// }
// }
// //获取消息头的信息
Header[] headers = response.getAllHeaders();
for (int i = 0; i < headers.length; i++) {
System.out.println(headers[i]);
}


get("http://www.gldjc.com");


// http://www.gldjc.com/user/auth/cas/callback?url=http%3A%2F%2Fwww.gldjc.com%2F&ticket=ST-4ce263ae06717a4ad64ddb39aad95
post.abort();


}


static void login2(String user, String pwd, boolean debug)
throws IOException {
HttpPost post = new HttpPost("http://www.wocai.net/login/save");
post.setHeader(
"User-Agent",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3");


// 登录表单的信息
List<NameValuePair> qparams = new ArrayList<NameValuePair>();
qparams.add(new BasicNameValuePair("username", user));
qparams.add(new BasicNameValuePair("password", pwd));


UrlEncodedFormEntity params = new UrlEncodedFormEntity(qparams, "UTF-8");
post.setEntity(params);


// Execute the request
HttpResponse response = client.execute(post);


if (debug) {
// Examine the response status
System.out.println(response.getStatusLine());


// Get hold of the response entity
HttpEntity entity = response.getEntity();
System.out.println("server return code:"
+ response.getStatusLine().getStatusCode());
dump(entity);
}
post.abort();
}


/**
* 打印页面
* 
* @param entity
* @throws IOException
*/
private static void dump(HttpEntity entity) throws IOException {
BufferedReader br = new BufferedReader(new InputStreamReader(
entity.getContent(), "UTF-8"));


System.out.println(IOUtils.toString(br));
}


/**
* 
* <一句话描述功能>生成数据</一句话描述功能>
* 
* @param exportList
* @return
*/
public static HSSFWorkbook exportProductPrice(
List<ProductPrice> exportList, int start, int max) {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss");
// 声明一个工作薄
HSSFWorkbook workbook = new HSSFWorkbook();
// 生成一个表格
HSSFSheet sheet = workbook.createSheet("供应商报价");
sheet.setColumnWidth(0, 18 * 128);
sheet.setColumnWidth(1, 18 * 512);
sheet.setColumnWidth(2, 18 * 512);
sheet.setColumnWidth(3, 18 * 512);
sheet.setColumnWidth(4, 18 * 256);
sheet.setColumnWidth(5, 18 * 256);
sheet.setColumnWidth(6, 18 * 256);
sheet.setColumnWidth(7, 18 * 256);
sheet.setColumnWidth(8, 18 * 256);
// 生成一个样式
HSSFCellStyle style = workbook.createCellStyle();
// 设置这些样式
style.setFillForegroundColor(HSSFColor.SKY_BLUE.index);
style.setFillPattern(HSSFCellStyle.SOLID_FOREGROUND);
style.setBorderBottom(HSSFCellStyle.BORDER_THIN);
style.setBorderLeft(HSSFCellStyle.BORDER_THIN);
style.setBorderRight(HSSFCellStyle.BORDER_THIN);
style.setBorderTop(HSSFCellStyle.BORDER_THIN);
style.setAlignment(HSSFCellStyle.ALIGN_CENTER);
// 生成一个字体
HSSFFont font = workbook.createFont();
font.setColor(HSSFColor.VIOLET.index);
font.setFontHeightInPoints((short) 12);
font.setBoldweight(HSSFFont.BOLDWEIGHT_BOLD);
// 把字体应用到当前的样式
// style.setFont(font);


HSSFCellStyle styledata = workbook.createCellStyle();
styledata.setAlignment(HSSFCellStyle.ALIGN_CENTER);


HSSFRow row = sheet.createRow(0);
// 1列
HSSFCell cell = row.createCell(0);
cell.setCellStyle(style);
cell.setCellValue("公司名称");
//
cell = row.createCell(1);
cell.setCellStyle(style);
cell.setCellValue("产品分类");
// 3列
cell = row.createCell(2);
cell.setCellStyle(style);
cell.setCellValue("产品名称");


cell = row.createCell(3);
cell.setCellStyle(style);
cell.setCellValue("规格/型号");


cell = row.createCell(4);
cell.setCellStyle(style);
cell.setCellValue("品牌");


cell = row.createCell(5);
cell.setCellStyle(style);
cell.setCellValue("计量单位");
//
cell = row.createCell(6);
cell.setCellStyle(style);
cell.setCellValue("报价");


cell = row.createCell(7);
cell.setCellStyle(style);
cell.setCellValue("地区");


cell = row.createCell(8);
cell.setCellStyle(style);
cell.setCellValue("报价时间");
int j = 0;
if (exportList != null && exportList.size() > 0) {
for (int i = start * 35000; i < exportList.size(); i++) {
if (i > max) {
break;
}
ProductPrice price = exportList.get(i);
if (price != null && price.getCity() != null
&& price.getProduct() != null
&& price.getProduct().getBrand() != null
& price.getProduct().getCompany() != null) {
HSSFRow content = sheet.createRow(j + 1);
cell = content.createCell(0);
cell.setCellValue(price.getProduct().getCompany().getName());
cell = content.createCell(1);
cell.setCellValue(price.getProduct().getCategory()
.getTitle());
cell = content.createCell(2);
cell.setCellValue(price.getProduct().getName());
cell = content.createCell(3);
cell.setCellValue(price.getProduct().getFormat());
cell = content.createCell(4);
cell.setCellValue(price.getProduct().getBrand().getName());
cell = content.createCell(5);
cell.setCellValue(price.getProduct().getMeasuring());
cell = content.createCell(6);
cell.setCellValue(price.getPrice());
cell = content.createCell(7);
cell.setCellValue(price.getCity().getName());
cell = content.createCell(8);
cell.setCellValue(format.format(price.getCreated()));
j++;
}


}
}


return workbook;
}


public static HSSFWorkbook testexportProductPrice(
List<ProductPrice> exportList, int start, int max) {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH-mm-ss");
// 声明一个工作薄
HSSFWorkbook workbook = new HSSFWorkbook();
// 生成一个表格
HSSFSheet sheet = workbook.createSheet("供应商报价");
sheet.setColumnWidth(0, 18 * 128);
sheet.setColumnWidth(1, 18 * 512);
sheet.setColumnWidth(2, 18 * 512);
sheet.setColumnWidth(3, 18 * 512);
sheet.setColumnWidth(4, 18 * 256);
sheet.setColumnWidth(5, 18 * 256);
sheet.setColumnWidth(6, 18 * 256);
sheet.setColumnWidth(7, 18 * 256);
sheet.setColumnWidth(8, 18 * 256);
// 生成一个样式
HSSFCellStyle style = workbook.createCellStyle();
// 设置这些样式
style.setFillForegroundColor(HSSFColor.SKY_BLUE.index);
style.setFillPattern(HSSFCellStyle.SOLID_FOREGROUND);
style.setBorderBottom(HSSFCellStyle.BORDER_THIN);
style.setBorderLeft(HSSFCellStyle.BORDER_THIN);
style.setBorderRight(HSSFCellStyle.BORDER_THIN);
style.setBorderTop(HSSFCellStyle.BORDER_THIN);
style.setAlignment(HSSFCellStyle.ALIGN_CENTER);
// 生成一个字体
HSSFFont font = workbook.createFont();
font.setColor(HSSFColor.VIOLET.index);
font.setFontHeightInPoints((short) 12);
font.setBoldweight(HSSFFont.BOLDWEIGHT_BOLD);
// 把字体应用到当前的样式
// style.setFont(font);


HSSFCellStyle styledata = workbook.createCellStyle();
styledata.setAlignment(HSSFCellStyle.ALIGN_CENTER);


HSSFRow row = sheet.createRow(0);
// 1列
HSSFCell cell = row.createCell(0);
cell.setCellStyle(style);
cell.setCellValue("公司名称");
//
cell = row.createCell(1);
cell.setCellStyle(style);
cell.setCellValue("产品分类");
// 3列
cell = row.createCell(2);
cell.setCellStyle(style);
cell.setCellValue("产品名称");


cell = row.createCell(3);
cell.setCellStyle(style);
cell.setCellValue("规格/型号");


cell = row.createCell(4);
cell.setCellStyle(style);
cell.setCellValue("品牌");


cell = row.createCell(5);
cell.setCellStyle(style);
cell.setCellValue("计量单位");
//
cell = row.createCell(6);
cell.setCellStyle(style);
cell.setCellValue("报价");


cell = row.createCell(7);
cell.setCellStyle(style);
cell.setCellValue("地区");


cell = row.createCell(8);
cell.setCellStyle(style);
cell.setCellValue("报价时间");
int j = 0;
System.out.println(j + "-" + start);
if (exportList != null && exportList.size() > 0) {
for (int i = start * 35000; i < exportList.size(); i++) {
if (i > max) {
break;
}
ProductPrice price = exportList.get(i);
HSSFRow content = sheet.createRow(j + 1);
// System.out.println(j + 1);
cell = content.createCell(0);
cell.setCellValue("测试钢筋");
cell = content.createCell(1);
cell.setCellValue("测试钢筋名称");
cell = content.createCell(2);
cell.setCellValue("test");
cell = content.createCell(3);
cell.setCellValue("test");
cell = content.createCell(4);
cell.setCellValue("test");
cell = content.createCell(5);
cell.setCellValue("test");
cell = content.createCell(6);
cell.setCellValue("test");
cell = content.createCell(7);
cell.setCellValue("test");
cell = content.createCell(8);
cell.setCellValue("test");
j++;


}
}


return workbook;
}


/**
* 
* <一句话描述功能>导出供应商报价</一句话描述功能>
*/
public static void exportPrice(List<ProductPrice> productprice) {
// for(int j=0;j<999999;j++){
// ProductPrice p=new ProductPrice();
// productprice.add(p);
// }
String dom = StringUtils.replace(domain, "http://", "");
dom = StringUtils.replace(dom, "/", "");
int dataIndex = productprice.size();
double sheetIndex = dataIndex / 35000 + 1;
System.out.println("总数" + dataIndex);
for (int i = 1; i <= sheetIndex; i++) {
long s = System.currentTimeMillis();
String name = dom + i + "__" + s + ".xls";
HSSFWorkbook workbook = exportProductPrice(productprice, i - 1,
i * 35000);
// HSSFWorkbook workbook = testexportProductPrice(productprice, i -
// 1,
// i * 35000);
try {
FileOutputStream outputStream = new FileOutputStream(
"e://prodcut//" + name);
workbook.write(outputStream);
outputStream.flush();
outputStream.close();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}