chrome 无头浏览器爬代理网站

from selenium import webdriver
import pymysql


# 创建chrome参数对象
opt=webdriver.ChromeOptions()
# 把chrome设置成无界面模式，不论windows还是linux都可以，自动适配对应参数
opt.set_headless()
# 创建chrome无界面对象
driver=webdriver.Chrome(options=opt)
driver.get('http://www.xicidaili.com/')

type=driver.find_elements_by_xpath('//*[@id="ip_list"]/tbody/tr/td[6]')
type=[i.text.lower() for i in type]

ip=driver.find_elements_by_xpath('//*[@id="ip_list"]/tbody/tr/td[2]')
ip=[i.text for i in ip]

port=driver.find_elements_by_xpath('//*[@id="ip_list"]/tbody/tr/td[3]')
port=[i.text for i in port]



conn = pymysql.connect(host = "localhost", port = 3306, user = "root", password = "liuzhiyu", db = "python77", charset = "utf8")
sql='insert into proxyServer(ipType,ipAddress,ipPort) values(%s,%s,%s)'
cursor = conn.cursor()
for i in range(len(ip)):
    print('正在插入地{}条'.format(i))
    cursor.execute(sql, (type[i],ip[i],port[i]))
conn.commit()
cursor.close()
conn.close()