python selenium sqlalchemy pyquery 进行数据爬虫

根据浏览器的版本，下载webdriverhttp://npm.taobao.org/mirrors/chromedriver/
安装selenium

pip install selenium

只给demo，不讲解哈，有时间会出一个selenium的详细讲解博客
dataBase.py

from sqlalchemy.ext.declarative import declarative_base
import sqlalchemy
from sqlalchemy.orm import sessionmaker,scoped_session

class connects():
    def __init__(self,user,password,db,host="localhost",port=5432):
        self.user=user
        self.password=password
        self.db=db
        self.host=host
        self.port=port

    def connectmethod(self):
        url='postgresql://{}:{}@{}:{}/{}'
        url=url.format(self.user,self.password,self.host,self.port,self.db)

        con=sqlalchemy.create_engine(url,client_encoding="utf-8")

        return con

conn=connects('postgres','postgres','postgres').connectmethod()
Base=declarative_base()
Base.metadata.bind=conn
Dbsession=scoped_session(sessionmaker(bind=conn))

table.py

from dataBase import Base
from sqlalchemy import String,Column
class kenny(Base):
    __tablename__="kenny"
    id=Column(String,primary_key=True)
    number=Column(String)

    def __init__(self,id,number):
        self.id=id
        self.number=number

main.py

from selenium import webdriver

from selenium.webdriver.common.by import By

from selenium.webdriver.support import  expected_conditions as EC

from selenium.webdriver.support.ui import WebDriverWait

from pyquery import PyQuery as pq
import os
from dataBase import Dbsession
from table import kenny
import requests
from requests.auth import HTTPBasicAuth
from pyquery import PyQuery as pq
import time

from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

desired_capabilities = DesiredCapabilities.INTERNETEXPLORER  # 修改页面加载策略,防止程序卡住，一直加载页面

desired_capabilities["pageLoadStrategy"] = "none"

driver = webdriver.Chrome()
driver.get("https://3&id=47763223")

WebDriverWait(driver,50,1).until(EC.presence_of_element_located((By.XPATH,'//*[@id="loginForm"]/div[1]/div[1]/div/span[2]')))

driver.find_element_by_xpath('//*[@id="loginForm"]/div[1]/div[1]/div/span[2]').click()



WebDriverWait(driver,50,1).until(EC.presence_of_element_located((By.XPATH,'//*[@id="test-execution-tree-3-47763728"]/span[4]')))



for i in range(0,1000,1):
    try:
        count=47763224+i
        buto=f'//*[@id="test-execution-tree-3-{count}"]/span[4]'
        driver.find_element_by_xpath(buto).click()
        WebDriverWait(driver, 50, 1).until(
            EC.presence_of_element_located((By.XPATH, '//*[@id="executionHistoryGrid_testnumber_header_executionHistoryGrid_tester_header_executionHistoryGrid_name_header_executionHistoryGrid_testCaseVersion_header_executionHistoryGrid_targetReleaseBuild_header_executionHistoryGrid_executionTypeDoNotUse_header_executionHistoryGrid_plannedStartDate_header_executionHistoryGrid_plannedEndDate_header_executionHistoryGrid_executedStart_header_executionHistoryGrid_executedEnd_header_executionHistoryGrid_status_header_executionHistoryGrid_defect_header_executionHistoryGrid_actions_header_"]/tbody')))
        time.sleep(2)
        test=pq(driver.page_source)
        s2 = pq(pq(test('tbody')[3])('tr')[0]).attr('data-objid')
        tab=kenny(str(count),str(s2))
        Dbsession.add(tab)
        Dbsession.commit()
        print(pq(s2))
    except:
        pass