java selenium 抓包_JAVA Selenium HO 数据抓取 可供参考其他抓取，大同小异

实现页面抓取:

/**

*

*/

package com.trafree.workfactory.service;

import java.util.List;

import org.apache.curator.shaded.com.google.common.base.Objects;

import org.openqa.selenium.By;

import org.openqa.selenium.JavascriptExecutor;

import org.openqa.selenium.Keys;

import org.openqa.selenium.WebDriver;

import org.openqa.selenium.WebElement;

import com.trafree.hardcore.util.MyWebdriver;

/**

* @author MichaelYang

*

*/

public class HOService {

public static void main(String[] args) {

//页面URL

String HOUrl="http://www.juneyaoair.com/";

//航班信息

int tripType = 2;//单程、往返、多程

String dep = "pvg";

String arr = "kmg";

String goDate = "2020-07-20";

String backDate = "2020-07-25";

int adtNum = 3;

WebDriver driver=MyWebdriver.getWebDriver();

try {

driver.get("http://www.juneyaoair.com/");

MyWebdriver.scrollToElement(driver,By.id("wangfan"));

switch(tripType) {

case 1:

{

MyWebdriver.executeScript(driver, "document.getElementById('dancheng').click();");

//出发地

WebElement fromcity=driver.findElement(By.id("fromcity"));

fromcity.click();

fromcity.clear();

fromcity.sendKeys(dep);

fromcity.sendKeys(Keys.ENTER);

//目的地

WebElement tocity=driver.findElement(By.id("tocity"));

tocity.click();

tocity.clear();

tocity.sendKeys(arr);

tocity.sendKeys(Keys.ENTER);

if(MyWebdriver.wait(driver, 1, "对不起，没有找到数据")>0) {

System.out.println("第二步验证该航线不支持");

}

//时间

WebElement depatureDate=driver.findElement(By.id("date01"));

depatureDate.click();

depatureDate.clear();

depatureDate.sendKeys(goDate);

//查询

WebElement search=driver.findElement(By.id("boxs-left-img-search"));

search.click();

if(MyWebdriver.isAlert(driver)) {

String msg=MyWebdriver.getAlertValue(driver);

if("请选择出发城市!".equals(msg)) {

System.out.println("该航线不支持");

}

if(MyWebdriver.wait(driver, 1, "您选择的日期没有航班信息，请重新选择！")>0) {

System.out.println("该航线和日期下无数据");

}

}

//等页面加载，不然会获取不到元素，报table没有数据

Thread.sleep(1500);

getTableInformation(driver,"flt_hd_table");

}

break;

case 2:

{

MyWebdriver.executeScript(driver, "document.getElementById('wangfan').click();");

//出发地

WebElement fromcity=driver.findElement(By.id("wfromCity"));

fromcity.click();

fromcity.clear();

fromcity.sendKeys(dep);

fromcity.sendKeys(Keys.ENTER);

//目的地

WebElement tocity=driver.findElement(By.id("wtoCity"));

tocity.click();

tocity.clear();

tocity.sendKeys(arr);

tocity.sendKeys(Keys.ENTER);

if(MyWebdriver.wait(driver, 1, "对不起，没有找到数据")>0) {

System.out.println("第二步验证该航线不支持");

}

//时间Dep

WebElement depatureDate=driver.findElement(By.id("date02"));

depatureDate.click();

depatureDate.clear();

depatureDate.sendKeys(goDate);

//时间Arr

WebElement arriveDate=driver.findElement(By.id("date03"));

arriveDate.click();

arriveDate.clear();

arriveDate.sendKeys(backDate);

//查询

WebElement search=driver.findElement(By.id("boxs-left-img-search1"));

search.click();

if(MyWebdriver.isAlert(driver)) {

String msg=MyWebdriver.getAlertValue(driver);

if("请选择出发城市!".equals(msg)) {

System.out.println("该航线不支持");

}

if(MyWebdriver.wait(driver, 1, "您选择的日期没有航班信息，请重新选择！")>0) {

System.out.println("该航线和日期下无数据");

}

}

//等页面加载，不然会获取不到元素，报table没有数据

Thread.sleep(1500);

//处理去程

driver.findElement(By.id("Go")).click();

getTableInformation(driver,"flt_hd_table");

//处理回程

driver.findElement(By.id("Back")).click();

if(MyWebdriver.isElementExistAndShow(driver, By.id("flt_hd_table_Back"))) {

getTableInformation(driver,"flt_hd_table_Back");

}

}

break;

case 3:

{

System.out.println("处理多程单程");

}

break;

case 4:

{

System.out.println("处理多程往返");

}

break;

default:

System.out.println("tripType 数值不对,请核验");

}

}catch(Exception e) {

e.printStackTrace();

System.out.println(e.getMessage()+"ERROR");

}finally {

//driver.quit();

}

}

private static void getTableInformation(WebDriver driver,String tableId) {

WebElement tableDiv=driver.findElement(By.id(tableId));

if(MyWebdriver.isElementExistAndShow(tableDiv, By.tagName("table"))) {

//获取数据

WebElement table=tableDiv.findElement(By.tagName("table"));

/*可以先将所有舱位展开再获取*/

List moreCabins=table.findElements(By.linkText("更多舱位"));

for(int i=0;i

WebElement link=moreCabins.get(i);

link.click();

}

WebElement tbody=table.findElement(By.tagName("tbody"));

List TRs=tbody.findElements(By.tagName("tr"));

List flightsOfTitle=table.findElements(By.className("title"));

System.out.println(flightsOfTitle.size());

//JavascriptExecutor driver_js=(JavascriptExecutor)driver;

//driver_js.executeScript("document.getElementsByClassName('cnt')");

int start=0;

if(flightsOfTitle.size()>0) {

//总共有多少个航班

for(int i=0;i

WebElement trTitle=flightsOfTitle.get(i);

String stopOverCode=trTitle.findElement(By.className("flt_arrow")).findElement(By.tagName("img")).getAttribute("stopairport");

String flightNum=trTitle.findElement(By.className("clearfix")).getText().split("\n")[0];

String depatureTime=trTitle.findElement(By.className("flt_from")).getText().split("\n")[0];

String depAirport=trTitle.findElement(By.className("flt_from")).getText().split("\n")[1];

String arriveTime=trTitle.findElement(By.className("flt_to")).getText().split("\n")[0];

String arrAirport=trTitle.findElement(By.className("flt_to")).getText().split("\n")[1];

String ticketFare=trTitle.findElement(By.className("flt_price")).findElement(By.tagName("b")).getText();

String ticketTax=trTitle.findElement(By.className("flt_tax")).findElement(By.tagName("a")).getText();

System.out.println(flightNum+" "+depatureTime+" "+depAirport+" "+arriveTime+" "+arrAirport+" "+ticketFare+" "+ticketTax+"**"+stopOverCode);

for(int j=start+1;j

WebElement TR=TRs.get(j);

if(TR.getAttribute("class").equals("title")){

continue;

}

if(TR.getAttribute("class").equals("flt_more")) {

start=j;

break;

}

//获取TD做处理即可

String cabin=TR.findElements(By.tagName("td")).get(0).getText();

System.out.println(cabin);

}

}

}else {

System.out.println("{table--title}没有数据!");

}

}else {

System.out.println("{table}没有数据!");

}

}

private void save2Db(Objects... args) {

}

}

实现页面登录:

/**

*

*/

package com.trafree.hardcore.login;

import java.util.concurrent.TimeUnit;

import org.openqa.selenium.By;

import org.openqa.selenium.WebElement;

import org.openqa.selenium.chrome.ChromeDriver;

import com.trafree.hardcore.util.MyWebdriver;

import com.trafree.hardcore.util.ScrollBarTool;

/**

* @author lily.li@trafree.com

*

*/

public class HOLogin {

public static String chromeDriverPath="C:\\Program Files (x86)\\ChromeCore\\chromedriver.exe";

private ChromeDriver driver;

public HOLogin() {

System.setProperty("webdriver.chrome.driver", chromeDriverPath);

driver= new ChromeDriver();

driver.manage().window().maximize();

driver.manage().timeouts().implicitlyWait(5, TimeUnit.SECONDS);

driver.manage().timeouts().pageLoadTimeout(10, TimeUnit.SECONDS);

}

public HOLogin(ChromeDriver driver) {

this.driver=driver;

}

public void loginHO(String username,String password) {

int indexPageCount=0;

driver.get("http://www.juneyaoair.com/");

int loginSwitch=MyWebdriver.wait(driver, 10, "登录");

if(loginSwitch>0) {

//进行登录操作

driver.findElement(By.id("LoginButton")).click();

int loginPageSwitch=MyWebdriver.wait(driver, 10, "会员登录");

if(loginPageSwitch==0) {

System.out.println("登录页面未加载,重新加载");

driver.findElement(By.id("LoginButton")).click();

}

driver.findElements(By.className("current")).get(0).click();

//账号

WebElement accountText=driver.findElement(By.id("accountName"));

accountText.clear();

accountText.click();

accountText.sendKeys(username);

//密码

WebElement passwordText=driver.findElement(By.id("pwd"));

passwordText.clear();

passwordText.click();

passwordText.sendKeys(password);

//点击已阅读条款

WebElement checkBox=driver.findElement(By.id("checkbox-account"));

checkBox.click();

//登录

WebElement loginButton=driver.findElement(By.id("accountLoginBtn"));

loginButton.click();

//拖动滑块验证

int isLogined=new ScrollBarTool().scrollBar(driver);

//判断

if(isLogined==0) {

System.out.println("登录成功");

}

}else {

if(indexPageCount>2) {

//尝试加载两次

System.out.println("停止加载");

return;

}

System.out.println("页面未加载成功，重新加载登录");

loginHO(username,password);

indexPageCount++;

}

}

public static void main(String[] args) {

new HOLogin().loginHO("13751115293", "michael0823");

}

}