selenium爬取某家生活分类平台

from selenium import webdriver
import pickle
import requests
from lxml import etree
from fake_useragent import UserAgent

base_url='https://passport.58.com/login/'
browser = webdriver.Firefox()
browser.get(base_url)
browser.implicitly_wait(10)

#切换密码登录
submit1=browser.find_element_by_class_name('pwdlogin')
browser.execute_script("arguments[0].click();", submit1)
browser.implicitly_wait(20)

#切换到账号密码登录方式

#输入账号
submit2=browser.find_element_by_id('usernameUser').send_keys('15958841269')
browser.implicitly_wait(20)

#输入密码,隐藏标签的提取
js = 'document.getElementById("passwordUser").style.display="block";'
browser.execute_script(js)
browser.implicitly_wait(20)
submit3=browser.find_element_by_id('passwordUser').send_keys('19880222ze')
browser.implicitly_wait(20)

#点击登录
submit =browser.find_element_by_class_name('submit')
browser.execute_script("arguments[0].click();", submit)
browser.implicitly_wait(10)

try:
    browser.find_element_by_class_name('uc-uname')
except:
    print('登录失败')

#保存cookies
pickle.dump(browser.get_cookies(),open("cookies.pkl", "wb"))
#刷新浏览器
browser.refresh()
browser.implicitly_wait(20)

#读取cookies流
cookies = pickle.load(open("cookies.pkl", "rb"))
#构造cookies
for cookie in cookies:
    cookie_dict = {
      # domain火狐浏览器不用填写，谷歌要需要
     'name': cookie.get('name'),
     'value': cookie.get('value'),
     "expires": "",
     'path': '/',
     'httpOnly': False,
     'HostOnly': False,
     'Secure': False}
     #添加cookies
    browser.add_cookie(cookie_dict)

browser.implicitly_wait(20)

headers={
    'User-Agent':UserAgent().random
}
start_urls = ['https://bj.58.com/zhuanye/pn{}'.format(num) for num in range(1,2)]


# 提取职位详细信息的链接地址
with open('bj.csv', 'a', encoding='utf-8') as f:
    for start_url in start_urls:

        response=requests.get(start_url,headers=headers)
        #获取职位信息链接
        e=etree.HTML(response.text)
        detail_urls = e.xpath('//div[@id="jingzhun"]/a/@href')
        print(detail_urls)
        for detail_url in detail_urls:
            #访问目标页面
            browser.get(detail_url)
            browser.implicitly_wait(20)
            #如果没有电话沟通的按钮直接跳过执行下一个循环
            if not browser.find_element_by_class_name("pos_operate_item.showPhoneBtn"):
                continue
            # 点击申请
            # 对是否申请过进行判断,如果申请过就不能再申请
            try:
                if browser.find_element_by_class_name('btn.btn-orange.btn-large.applyJobBtn'):

                    browser.find_element_by_class_name('btn.btn-orange.btn-large.applyJobBtn')
                    submit11 =browser.find_element_by_class_name('btn.btn-orange.btn-large.applyJobBtn')
                    browser.execute_script("arguments[0].click();", submit11)
                    browser.implicitly_wait(20)
                    #关闭提示框
                    submit12 = browser.find_element_by_id('close-guide')
                    browser.execute_script("arguments[0].click();", submit12)
                    browser.implicitly_wait(20)
                    #电话

                    submit13 = browser.find_element_by_class_name('pos_operate_item.showPhoneBtn')
                    browser.execute_script("arguments[0].click();", submit13)

            except:
                    # 电话
                    submit13 = browser.find_element_by_class_name('pos_operate_item.showPhoneBtn')
                    browser.execute_script("arguments[0].click();", submit13)


            comp_name = browser.find_element_by_xpath('//div[@class="baseInfo_link"]/a').get_attribute('textContent')
            f.write(comp_name + ' ')
            #提取老师信息
            teacher=browser.find_element_by_xpath('//div[@class="tan"]/p').get_attribute('textContent')
            f.write(teacher + ' ')
            try:
                browser.find_element_by_class_name("privaNum")
                tel=browser.find_element_by_xpath('//ul[@class="privaNum"]/li').get_attribute('textContent')

                f.write(tel + '\n')
            except:
                f.write('\n')