python爬取头条视频_python 爬取头条视频

知识点总结

1. 利用webdriver 模拟浏览器访问

from selenium import webdriver

2.import requests

3. from bs4 import BeautifulSoup

简单小例

import requests

from selenium import webdriver

from urllib.parse import urlencode

from bs4 import BeautifulSoup

def get_one_page():

# headers = {

#

# 'Referer': 'https://www.365yg.com/',

# 'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36",

# }

da = {

'min_behot_time': '0',

'category': 'video_new',

'utm_source': 'toutiao',

'widen': '1',

'tadrequire': 'true',

'as': 'A1654C1827C2B37',

'cp': '5C87724B93A76E1',

'_signature': ' o54nnxAd.ygc6NZ537gIfKOeJ4'

}

url = 'https://www.365yg.com/?'+urlencode(da)

opetions = webdriver.ChromeOptions()

browser = webdriver.Chrome(executable_path="D:/chromedriver_win32/chromedriver.exe")

# cookie={

#

# 'name': 'tt_webid',

# 'value': '6671039337541174792;',

# }

# browser.add_coo