实战selenium框架下在爱企查爬取企业的历史变更信息

weixin_45363959

已于 2024-06-20 16:12:31 修改

阅读量887

点赞数 4

文章标签： selenium chrome 爬虫

于 2024-05-15 10:30:34 首次发布

本文链接：https://blog.csdn.net/weixin_45363959/article/details/138897958

版权

代码如下

# 导入包
import random
import time
from tkinter import filedialog
import tkinter as tk
import xlrd
import os
import datetime
import csv
from selenium import webdriver
from selenium.webdriver import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC



def chooseFile():
    # 实例化
    root = tk.Tk()
    root.withdraw()
    # 获取文件夹路径
    f_path = filedialog.askopenfilename(title='选择公司列表文件', filetypes=[('xlsx', '*.XLSX'), ('xls', '*.xls')])
    if f_path == '':
        return f_path
    print('\n获取的文件地址：', f_path)
    # 打开文件
    with xlrd.open_workbook(f_path, formatting_info=False) as workbook:
        sheet1 = workbook.sheet_by_index(0)
    return f_path


def DList(tagList):
    nList = []
    if len(tagList) == 0:
        nList.append(" ")
        nList.append(" ")
        nList.append(" ")
    else:
        nList.append(tagList[0])
        nList.append(tagList[1])
        nList.append(tagList[2])

    return nList


def main():
    # 文件路径
    # xlsFile = "data//default_data_file.xlsx"
    xlsFile = chooseFile()
    if xlsFile == '':
        return
        # 打开文件
    workbook = xlrd.open_workbook(xlsFile, formatting_info=False)
    sheet1 = workbook.sheet_by_index(0)
    iIndex = 0
    allUrls = []

    while iIndex < sheet1.nrows:
        commodityId = str(sheet1.cell_value(iIndex, 0))
        allUrls.append([commodityId])
        iIndex = iIndex + 1

    currentDir = os.getcwd()  # "C:/scrapy" #os.path.dirname(os.path.abspath(__file__))
    dirName = currentDir + "//ouput//"
    curr_time = datetime.datetime.now()
    strNow = datetime.datetime.strftime(curr_time, '%Y%m%d%H%M%S')
    dirName = dirName + strNow
    os.makedirs(dirName)
    outCsv = dirName + "/out.csv"
    # region 定义浏览器
    # http请求头
    userAgent = [
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.289 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0",
        "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.289 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 SE 2.X MetaSr 1.0",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36 SLBrowser/9.0.3.1311 SLBChan/128",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
        "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.289 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 SE 2.X MetaSr 1.0"]
    # 指定谷歌的位置-- 本项目目录内，如果机器已经安装了Google浏览器。需要先卸载掉
    chrome_location = r'Chrome-bin\chrome.exe'
    # 定义options对象
    options = webdriver.ChromeOptions()
    options.binary_location = chrome_location
    # 1、禁止浏览器自动关闭
    options.add_experimental_option('detach', True)
    # 2、禁用测试控制
    options.add_experimental_option('excludeSwitches', ['enable-automation']

最低0.47元/天解锁文章