Selenium + Headless Chrome + 爬取动态网页（踩坑）

科研项目需要在云端做爬虫爬取YouTube的评论，我先在Google Colab试水，需要用加入Headless参数，并且设置窗口大小（很关键），查了很多资料才发现是没有设置窗口大小。

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException

options = Options()
options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--window-size=1920x1080")
options.add_argument("start-maximised")
driver = webdriver.Chrome(chrome_options=options)
driver.get('https://www.youtube.com/watch?v=yIYKR4sgzI8')

driver.execute_script('window.scrollTo(1, 500);')
time.sleep(5)

views_div = driver.find_element_by_xpath('//*[@id="info-contents"]')
views = views_div.find_element_by_xpath('//*[@class="view-count style-scope yt-view-count-renderer"]')

comment_div=driver.find_element_by_xpath('//*[@id="contents"]')
comments=comment_div.find_elements_by_xpath('//*[@id="content-text"]')

print(views.text)
for comment in comments:
    print(comment.text)

driver.close()

https://stackoverflow.com/questions/48257870/headless-chrome-with-selenium-can-only-find-ways-to-scroll-non-headless
https://testerhome.com/topics/15065
https://liaobu.de/python-selenium-chrome-headless-implements-automatically-present-gift-in-huya/