requests和selenium

requests获取cookie

from fake_useragent import UserAgent
import requests
 
def get_Cookies0():
    ua = UserAgent().ie  #随机获取请求头
    headers = {'User-Agent':ua}
    url = 'https://www.lagou.com/jobs/list_python?'
    session = requests.session()
    session.post(url,headers = headers)
    cookie = session.cookies
    a = cookie.get_dict()
    print (a)

from fake_useragent import UserAgent
import requests
def get_Cookies1():
    ua = UserAgent().ie
    headers = {'User-Agent': ua}
    url = 'https://www.lagou.com/jobs/list_python?'
    res = requests.get(url,headers=headers)
    cookiejar = res.cookies
    cookiedict = requests.utils.dict_from_cookiejar(cookiejar)
    print (cookiejar)
    print(cookiedict)

selenium获取cookie，添加cookie

#添加cookie
driver.get("https://www.ketangpai.com/User/login.html")
for key, value in cookies_dict.items():
    browser.add_cookie({'name': key, 'value': value})
driver.get("https://www.ketangpai.com/Main/index.html")
# 获取cookie
driver1.get_cookies()

防止网站识别Selenium代码（针对老版本）

from selenium.webdriver import Chrome
from selenium.webdriver import ChromeOptions

option = ChromeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
bro = Chrome(options=option)

url = "fudan.bbs.kaoyan.com"  # 首页
bro.get("http://fudan.bbs.kaoyan.com/")
bro.implicitly_wait(10)

新版本如下：

from selenium.webdriver import Chrome
from selenium.webdriver import ChromeOptions

options = ChromeOptions()
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option('useAutomationExtension', False)

bro = Chrome(options=options)
bro.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
  "source": """
    Object.defineProperty(navigator, 'webdriver', {
      get: () => undefined
    })
  """
})

url = "fudan.bbs.kaoyan.com"  # 首页
bro.get("http://fudan.bbs.kaoyan.com/")
bro.implicitly_wait(10)