使用selenium+pyquery爬取豆瓣top250,并保存数据库中(这里写自定义目录标题)
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
import os
import time
from selenium.webdriver.common.keys import Keys
from pyquery import PyQuery as pq
import pymysql
chromedriver = "E:\chromeDriver\chromedriver.exe"
os.environ["webdriver.chrome.driver"] = chromedriver
driver = webdriver.Chrome(chromedriver)
driver.maximize_window()
start_url = "https://movie.douban.com/top250"
for i in range(10):
url = start_url+'?start='+str(25*i)+"&filter="
driver.get(url)
time.sleep(2)
html = driver.page_source
doc = pq(html)
list = []
for t in doc(".grid_view li"):
name = pq(t).find('.info .hd .title').text()
daoyan = pq(t).find(".info .bd p").text()
pingjia = pq(t).find(".star span").text()
list.append([name,daoyan,pingjia])
db = pymysql.connect(host="localhost", user="root", password="rootroot", database="pachong")
cursor = db.cursor()
for i in list:
sql = """INSERT INTO movie(name,
daoyan, pingjia)
VALUES (%s, %s,%s)"""
try:
cursor.execute(sql,i)
db.commit()
except:
db.rollback()
db.close()