jsoup的jar包maven导入
<dependencies>
<!-- 爬虫-->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.13.1</version>
</dependency>
</dependencies>
爬取的内容打印到控制台
你也可以输出到文件
在输出到文件方法中的注释就是输出到文件,你修改一下文件路径和文件的格式就可以了。
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.*;
import java.util.HashMap;
import java.util.Map;
/**
* @author kliker
* @version 1.0
*/
//生成文档注释 javadoc -d F:\SSM\data -author -version Main.java
public class Main {
private static Map<String,String> map = new HashMap<>();
public static void main(String[] args) {
for(int i = 1;i<101;i++) {
String url = "https://sz.lianjia.com/zufang/pg"+i+"rt200600000002/#contentList";
try {
Document html = Jsoup.connect(url).get();
Elements elements = html.getElementsByClass("twoline");
for (Element element : elements) {
String title = element.text();
String content = element.getElementsByTag("a").attr("href");
// System.out.println(title + "\t" + "https://sz.lianjia.com" + content);
String contents = getHtml(content);
map.put(title,contents);
}
writeFile(map);
} catch (IOException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
}
}
//获取房子详细信息
public static String getHtml(String content){
String url = "https://sz.lianjia.com"+content;
String contents1 = null;
String contents2 = null;
try {
Document html = Jsoup.connect(url).get();
Elements elements1 = html.select("#info");
// Elements elements2 = html.select("body > div.wrapper > div:nth-child(2) > div.content.clear.w1150 > div.content__detail > div.content__article.fl > ul");
// Elements elementsImg = html.select("#gslide > div > p.gslide--list");
// for(Element element : elementsImg){
// String attr = element.getElementsByTag("img").attr("src");
// getImg(attr);
// }
contents1 = elements1.text();
// contents2 = elements2.text();
System.out.println(contents1);
}catch(Exception e){
e.printStackTrace();
}
return contents1;
}
//获取房子图片
// public static void getImg(String urlImg){
// try {
// Document html = Jsoup.connect(urlImg).get();
// }catch(Exception e){
// e.printStackTrace();
// }
// }
//写入文件
public static void writeFile(Map<String,String> map) {
try {
// BufferedWriter bf = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(new File("data/shenzhenzufang.txt"))));
// for(Map.Entry<String,String> entry:map.entrySet()) {
// bf.write(entry.getKey());
// bf.newLine();
// bf.write(entry.getValue());
// bf.newLine();
// }
// bf.flush();
// bf.close();
PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.out));
for(Map.Entry<String,String> entry:map.entrySet()) {
pw.println(entry.getKey());
pw.println(entry.getValue());
}
}catch(Exception e){
e.printStackTrace();
}
}
}