java 爬虫Demo

package com.iminer.crawlers.gsdata;

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.List;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.UnexpectedPage;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.iminer.crawlers.CrawlerUtils4MusicAndMovie;

public class Test {
	
	static String url = "http://www.gsdata.cn/index.php/rank/ranks?gid=@GID@&date=2015-09-22&page=1";
	static String detailurl = "http://www.gsdata.cn/index.php/rank/single?id=";

	
	public static void main(String[] args) throws Exception {
		WebClient webClient = CrawlerUtils4MusicAndMovie.getClient();
		String entranceUrl  ="http://www.gsdata.cn/index.php/rank/detail?gid=0";
		HtmlPage page = webClient.getPage(entranceUrl);
		List<?> byXPath = page.getByXPath("//ul[@class='group-items']/li");
		List<String> gids = new ArrayList<String>();
		
		for (Object object : byXPath) {
			HtmlElement oElement = (HtmlElement) object;
			String attribute = oElement.getAttribute("data-gid");
			gids.add(attribute);
		}

		for (String gid : gids) {
			String tempUrl  = url.replace("@GID@", gid);
			webClient.addRequestHeader("X-Requested-With", "XMLHttpRequest");
			UnexpectedPage spage = webClient.getPage(tempUrl);
			JSONObject jsonObject = JSONObject.fromObject(spage.getWebResponse().getContentAsString());
			String total = jsonObject.getJSONObject("data").getString("total");
			System.out.println(total);
			JSONArray jsonArray = jsonObject.getJSONObject("data").getJSONArray("rows");
			for (Object object : jsonArray) {
				JSONObject jsonObject2 = (JSONObject) object;
				String id = jsonObject2.getString("nickname_id");
				String temptempurl = detailurl + id;
				System.out.println(temptempurl);
				
				//详细页的访问
				HtmlPage page2 = webClient.getPage(temptempurl);
				List<?> byXPath2 = page2.getByXPath("//li[@class='li_2']");
				for (Object object2 : byXPath2) {
					HtmlElement element = (HtmlElement) object2;
					System.out.println(element.getTextContent());
				}
				
				//访问统计数据
				//http://www.gsdata.cn/index.php/rank/singleStatistic?id=52
				
				
			}
			
			
			
		}
		
		

		
		
		
		
	}
	
}