爬虫爬取点击动态ajax请求的数据方法

先下载geckodriver.exe 驱动
下面展示一些 代码片。

// An highlighted block
@Test
    public void Test() throws Exception{
        // 加载驱动
       System.setProperty("webdriver.gecko.driver", "d://geckodriver.exe");
        FirefoxDriver driver = new FirefoxDriver();
       // 打开指定的网站
       String indexUrl = "https://yyk.99.com.cn/";
       driver.get(indexUrl);
       //定位层级元素 获取首页
       String indexhandle = driver.getWindowHandle();
       List<WebElement> areas = driver.findElements(By.cssSelector(".area-list .aere-txt2"));
       if (!CollectionUtils.isEmpty(areas)) {
           areas.stream().forEach(p->{
               String provinceName = p.getText();
               Actions actionOpenLinkInNewTab = new Actions(driver);


               String provinceUrl = p.getAttribute("href");

               p.click();
               sleep(driver,2000);
               //增加一个新的页签
               while (driver.getWindowHandles().size() == 1){
                   System.out.println("sleep0");
                   sleep(driver,500);
               }
               for (String temhandle : driver.getWindowHandles()) {    //获取所有的句柄，循环判断是否是新句柄
                   if (!temhandle.equals(indexhandle)) {
                       driver.switchTo().window(temhandle);
                   }
               }
//                driver.get(provinceUrl);
               //打开省页面
               String provincehandle = driver.getWindowHandle();
               List<WebElement> cityList = driver.findElements(By.cssSelector(".u-title-div a"));
               List<CityDto> cityDtoList = new ArrayList<>();
               while (CollectionUtils.isEmpty(cityList)){
                   sleep(driver,500);
                   cityList = driver.findElements(By.cssSelector(".u-title-div a"));
               }
               cityList.stream().forEach(pp->{
                   //进到市页面
                   //增加一个新的页签
                   Map map = new HashMap<>();
                   CityDto cityDto = new CityDto();
                   cityDto.setCityName(pp.getText());
                   cityDto.setCityUrl(pp.getAttribute("href"));
                   cityDtoList.add(cityDto);
               });

               cityDtoList.stream().forEach(cp->{
                   driver.get(cp.getCityUrl());
                   List<WebElement> elements = driver.findElements(By.cssSelector(".m-box"));
                   while (CollectionUtils.isEmpty(elements)){
                       sleep(driver,500);
                       elements = driver.findElements(By.cssSelector(".m-box"));
                       System.out.println("sleep1");
                   }
                   if (!CollectionUtils.isEmpty(elements)){
                       elements.remove(0);
                   }
                   elements.stream().forEach(ppp->{
                       List<Hospital99> list = new ArrayList<>();
                       List<WebElement> elements1 = driver.findElements(By.cssSelector(".u-title-3 span"));
                       String regionName = elements1.get(0).getText();
                       List<WebElement> hospitalElements = ppp.findElements(By.cssSelector(".m-table-2 a"));
                       for (WebElement hospitalElement : hospitalElements) {
                           hospitalElement.click();

                           while (driver.getWindowHandles().size() == 2){
                               try {
                                   Thread.sleep(500);
                                   System.out.println("sleep2");
                               } catch (InterruptedException e) {
                                   e.printStackTrace();
                               }
                           }
                           for (String temhandle : driver.getWindowHandles()) {    //获取所有的句柄，循环判断是否是新句柄
                               if (!temhandle.equals(indexhandle)&& !temhandle.equals(provincehandle)) {
                                   driver.switchTo().window(temhandle);
                               }
                           }
                           List<WebElement> hospitalRealNameElement = driver.findElements(By.cssSelector(".wrap-mn>h1"));
                           Integer count = 0;
                           while (CollectionUtils.isEmpty(hospitalRealNameElement)) {
                               try {
                                   if (count >= 5 ){
                                       System.out.println("refreshPage");
                                       count = 0;
                                       driver.navigate().refresh();
                                   }
                                   count++;
                                   Thread.sleep(500);
                                   hospitalRealNameElement = driver.findElements(By.cssSelector(".wrap-mn>h1"));
                                   System.out.println("sleep3");
                               } catch (InterruptedException e) {
                                   e.printStackTrace();
                               }
                           }
                           String hospitalRealName= "";
                           if (!CollectionUtils.isEmpty(hospitalRealNameElement)) {
                               hospitalRealName = hospitalRealNameElement.get(0).getText();
                           }
                           String hospitalLevel = "";
                           List<WebElement> hospitalLevelElement = driver.findElements(By.cssSelector(".wrap-grade .grade"));
                           if (!CollectionUtils.isEmpty(hospitalLevelElement)) {
                               hospitalLevel = hospitalLevelElement.get(0).getText();
                           }
                           List<WebElement> hospitalDto = driver.findElements(By.cssSelector(".wrap-info>dd>p"));

                           if (!CollectionUtils.isEmpty(hospitalDto)&&hospitalDto.size() == 4) {
                               String hospitalName = hospitalDto.get(0).getText().substring(3);
                               String hospitalType = hospitalDto.get(1).getText().substring(3);
                               String hospitalMobile = hospitalDto.get(2).getText().substring(3);
                               String hospitalAddress = hospitalDto.get(3).getText().substring(3);
                               String hospitalUrl = driver.getCurrentUrl();
                               Hospital99 hospital99 = new Hospital99();
                               hospital99.setHospitalName(hospitalName);
                               hospital99.setHospitalType(hospitalType);
                               hospital99.setMobile(hospitalMobile);
                               hospital99.setAddress(hospitalAddress);
                               hospital99.setCityName(cp.getCityName());
                               hospital99.setHospitalUrl(hospitalUrl);
                               hospital99.setCityUrl(cp.getCityUrl());
                               hospital99.setRegionName(regionName);
                               hospital99.setHospitalRealName(hospitalRealName);
                               hospital99.setHospitalLevel(hospitalLevel);
                               hospital99.setRegionUrl(cp.getCityUrl().substring(0,cp.getCityUrl().lastIndexOf('/')));
                               hospital99.setProvinceName(provinceName);
                               hospital99.setProvinceUrl(provinceUrl);
                               list.add(hospital99);
                               System.out.println(hospitalUrl+"------------------------------"+hospitalRealName);
                           }
                           //关闭医院 切回市
                           driver.close();
                           driver.switchTo().window(provincehandle);
                       }
                       System.out.println("==============" + JSON.toJSONString(list));
                   });

               });

               //关闭省 切回首页
               driver.close();
               driver.switchTo().window(indexhandle);
           });
           //首页关闭
           driver.close();
       }
       driver.quit();// 退出浏览器
   }

public static void sleep(WebDriver driver, long timemills){
        try {
            //等待界面加载完
            Thread.sleep(timemills);
            driver.manage().timeouts().implicitlyWait(10,TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

    }