实现用火狐进行动态网页抓取

最新推荐文章于 2021-05-31 16:47:07 发布

喵-

最新推荐文章于 2021-05-31 16:47:07 发布

阅读量1.9k

点赞数

文章标签： python3

本文链接：https://blog.csdn.net/qq_41231439/article/details/78889219

版权

# -*- coding: utf-8 -*-
import sys,re
from lxml import etree
import  time
from selenium import webdriver
import os
import pymysql

host = '服务器IP'
dbName = '数据库名'
user = '数据库登陆用户名'
password = '数据库登陆密码'
db = pymysql.connect(host, user, password, dbName, charset='utf8')
cursor = db.cursor()
cursor.execute("DROP TABLE IF EXISTS EMPLOYEE")  
sql = """CREATE TABLE EMPLOYEE (SYMBOL CHAR(6) NOT NULL,NAME CHAR(10),PRICE CHAR(10))"""
cursor.execute(sql)

driver = webdriver.Firefox()  #打开火狐浏览器
driver.set_window_position(x=70,y=30)   #设置火狐浏览器位置
driver.set_window_size(width=1064, height=524) #设置火狐浏览器窗口大小
f = open("symbol.txt", 'r')   # 以只读方式打开当前路径下的symbol.txt文件
for line in f.readlines():
   line = line.replace("\n", "")  # 提取每行数据，去掉换行符
   url = "url1"+line+"url2"  # url拼接
   driver.get(url)  #输入需要访问的URL
   time.sleep(3)    #等待3秒
   content = driver.page_source.encode('utf-8') #网页内容转码utf8
   html = etree.HTML(content)
   price = html.xpath('.//*[@class="ico_data col_open"]/text()') #提取价格
   name = html.xpath('.//*[@class="ico_name label_open"]/text()') #提取名称
   symbolname = ''.join(name)
   price = ''.join(price)
   print(line, symbolname, price)
   cursor = db.cursor()
   sql1 = ("INSERT INTO EMPLOYEE(SYMBOL, NAME, PRICE) VALUES ('%s', '%s', '%s')" % (str(line), symbolname, str(price)))
   cursor.execute(sql1)
   db.commit()  #提交数据
db.close() #关闭数据库
f.close()  #关闭symbol文件
driver.quit() #退出浏览器