【大数据头歌实验八】大数据综合实验 （旅游网站大数据分析 - 数据抓取）

第1关：利用Jsoup抓取携程旅游网的数据

package step1;
import java.io.File;
import java.io.IOException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

public class Task {
	/**
	 * @param filePath	文件路径：backups/www.ctrip.com.txt/
	 * @return
	 * @throws IOException
	 */
	public Document getHtml1(String url) throws IOException{        
        Document document = Jsoup.parse( new File( "./backups/www.ctrip.com.txt" ) , "utf-8" );
		   // System.out.println(document.title());
		   // System.out.println(document);
        
		return document;
	} 
	/**
	 * 
	 * @param url	网址http://hotels.ctrip.com/domestic-city-hotel.html
	 * @return
	 * @throws IOException
	 */
	public Document getHtml2(String url) throws IOException{
        Document document = Jsoup.parse( new File( "./backups/hotels.ctrip.com_domestic-city-hotel.txt" ) , "utf-8" );
        //System.out.println(document.title());
		return document;
	} 
}

第2关：解析并提取HTML 元素(一）

package step2;
import java.io.File;
import java.io.IOException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class Task {
    
 public Document getDoc1(String url) throws IOException{
      File file=new File("./backups/www.ctrip.com.txt");
       Document document =Jsoup.parse(file,"UTF-8","http://www.ctrip.com/");
       return document ;
    }
    //获取“http://you.ctrip.com/”的Docment对象
    public Document getDoc2(String url) throws IOException{
        File file=new File("./backups/you.ctrip.com.txt");
        Document document =Jsoup.parse(file,"UTF-8","http://you.ctrip.com");
        
        return document ;
    }
    //获取所有链接
    public Elements getLinks(Document doc){
       Elements links=doc.select("link[href]");
        return links;
    }
    //获取第一个class为“pop_attention”的div
    public Element getDiv(Document doc){
       Element element =doc.select("div.pop_attention").first();
        return element ;
    }
    //获取所有li之后的i标签
    public Elements getI(Document doc){
     Elements element =doc.select("li>i");
        return element ;
    }
}

第3关：解析并提取HTML 元素(二）

package step3;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class Task {
   
   //通过filePath文件路径获取Docment对象
   public Document getDoc(String filePath) throws IOException{
   	/**********   Begin   **********/
    File file=new File("./backups/hotel.ctrip.com.txt");
    Document doc=Jsoup.parse(file,"UTF-8","http://hotels.ctrip.com/");
    return doc;
   	/**********   End   **********/
   }

   //获取所有链接
   public List<String> getLinks(Document doc){
   	/**********   Begin   **********/
    List<String> ar=new ArrayList<>();
    Elements kk=doc.select("a[href]");
    for(Element gg:kk){
        ar.add(gg.tagName()+"$"+gg.attr("abs:href")+"("+gg.text()+")"); 
    } 
    return ar; 
   	/**********   End   **********/
   }
   //获取图片
   public List<String> getMedia(Document doc){
   	/**********   Begin   **********/
    List<String> list=new ArrayList<>(); 
    Elements ll=doc.select("[src]"); 
    for(Element h:ll){ 
        if(h.tagName().equals("img")){ 
            list.add(h.tagName()+"$"+h.attr("abs:src"));
        }
    }    
    return list;
   	/**********   End   **********/
   }
   
   //获取link[href]链接
   public List<String> getImports(Document doc){
   	/**********   Begin   **********/
    List<String> list=new ArrayList<>();
    Elements kk=doc.select("link[href]");
    for(Element g:kk)
    {
        list.add(g.tagName()+"$"+g.attr("abs:href")+"("+g.attr("rel")+")");
    }
    return list;
   	/**********   End   **********/
   }
   
}

第4关：使用Jsoup抓取携程旅游网全国城市信息

package step4;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class Task {
    
   public Document getDoc(String url) throws IOException{
        
      File file=new File("backups/hotels.ctrip.com_domestic-city-hotel.txt");
       Document doc=Jsoup.parse(file,"UTF-8","http://hotels.ctrip.com/");
        return doc;
	}
    
    /**
	 * 获取所有城市返回城市信息集合
	 * @param doc	
	 * @return
	 */
	public List<HotelCity> getAllCitys(Document doc){
       List<HotelCity> cities = new ArrayList<HotelCity>(); 
        
        Elements aa= doc.getElementsByClass("pinyin_filter_detail layoutfix");
        Element pp = aa.first();
        Elements hh= pp.getElementsByTag("dd");
        Elements hts=pp.getElementsByTag("dt");
        
       for (int i = 0; i < hh.size(); i++) {
        Element bb = hts.get(i);
        Element head_hotelsLink = hh.get(i);
        Elements links = head_hotelsLink.children();
           
        for (Element link : links) {
                String pinyin_cityId = link.attr("href").replace("/hotel/", "");
                String pinyin = pinyin_cityId.replace(StringUtil.getNumbers(link.attr("href")), "");//截取拼音
                HotelCity city = new HotelCity();
                city.setCityId(StringUtil.getNumbers(link.attr("href"))); //截取cityId
                city.setCityName(link.text());
                city.setHeadPinyin(bb.text());
                city.setPinyin(pinyin);
                cities.add(city);

    }
		
	}
        return cities;
    }

}