使用Jsoup和Selenium实现网页分析

使用Jsoup和Selenium实现网页分析

环境搭建

用idea创建一个maven项目。
在pom.xml配置文件中导入Jsoup和Selenium相关依赖。
例：
	<dependency>
      <groupId>org.seleniumhq.selenium</groupId>
      <artifactId>selenium-java</artifactId>
      <version>3.141.59</version>
    </dependency>
    <dependency>
      <groupId>org.jsoup</groupId>
      <artifactId>jsoup</artifactId>
      <version>1.11.3</version>
    </dependency>
下载浏览器对应的驱动
[Chome浏览器部分驱动下载](http://chromedriver.storage.googleapis.com/index.html)

代码示例

package org.example;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;

import java.util.Scanner;

/**
 * @Author: LHC
 * @Date: 2020/9
 * @Content:
 */
public class TestPrseMy {
    public static void main(String[] args) throws Exception {

        System.out.println("输入招聘查询关键词：");
        String str = new Scanner(System.in).nextLine();

        //设置浏览器驱动
        System.setProperty("webdriver.chrome.driver","D:\\MyExePrograms\\chromedriver_win32\\chromedriver.exe");//下载的chromedriver位置
        WebDriver driver = new ChromeDriver();//实例化ChromeDriver对象

        //网站地址
        String url="https://search.51job.com/list/090200,000000,0000,00,9,99,"+
                "java"+",2,50.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";
        //打开指定网站
        driver.get(url);
        //找到搜索输入框
        WebElement element1 = ((ChromeDriver) driver).findElementByCssSelector("#keywordInput");
        //内容清空
        element1.clear();
        //输入搜索关键词
        element1.sendKeys(str);
        //找到搜索按钮
        WebElement element2 = ((ChromeDriver) driver).findElementByCssSelector("#search_btn");
        //点击
        element2.click();
        //等待加载
        Thread.sleep(3000);
        /**
         * 用jsoup处理网页生成document对象
         * 分析网页结构
         * 可以选择需要信息用类封装
         */
        Document document = Jsoup.parse(driver.getPageSource());


        Elements ets = document.select(".e");
        for (int i = 0; i < ets.size(); i++) {
            String url1 = ets.get(i).getElementsByTag("a").eq(0).attr("href");
            String title = ets.get(i).getElementsByClass("jname").eq(0).text();
            String time = ets.get(i).getElementsByClass("time").eq(0).text();
            String sal = ets.get(i).getElementsByClass("sal").eq(0).text();
            String address = ets.get(i).getElementsByClass("d").eq(0).text();
            String tag = ets.get(i).getElementsByClass("tags").eq(0).text();
            String cname = ets.get(i).getElementsByClass("cname").eq(0).text();


            if (title.isEmpty())continue;

            System.out.print("职位\t"+title+"\t");
            System.out.print("网址\t"+url1);
            System.out.print("\t"+time);
            System.out.print("\t薪资: "+sal);
            System.out.print("\t相关信息： "+address);
            System.out.print("\t福利待遇： "+tag);
            System.out.println("\t公司名\t"+cname);
        }

    }
}