帮女朋友刷一下考研院校专业目录下载量,写个小爬虫,主要使用selenium,目标网址http://yz.nefu.edu.cn/info/1026/2189.htm
唯一难点是绕过验证码,直接使用ddddocr库
import ddddocr
from PIL import Image
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from io import BytesIO
import threading
time.sleep(1)
NUM_MAX = 500000 # 循环次数
PATH = 'C:\\Users\\yanga\\Desktop\\temp\\1.png'
url = 'http://yz.nefu.edu.cn/system/_content/download.jsp?urltype=news.DownloadAttachUrl&owner=1701911290&wbfileid' \
'=66AE325B0AE6939AB54B45742A479815 '
options = webdriver.ChromeOptions()
prefs = { # 禁用下载
"download_restrictions": 3,
}
options.add_experimental_option(
"prefs", prefs
)
browser = webdriver.Chrome(options=options) # 浏览器对象初始化
def recognize(): # 调用PATH路径图片并识别验证码
ocr = ddddocr.DdddOcr()
with open(PATH, 'rb') as f:
img_bytes = f.read()
res = ocr.classification(img_bytes)
print(res)
return res
def connect_web(): # 连接网页
try:
browser.get(url)
except TimeoutException:
print('time out!')
def ocr_img(): # 获取网页截图并裁剪至只有验证码,保存至PATH路径
temp_p = browser.get_screenshot_as_png()
big_image = Image.open(BytesIO(temp_p))
box = (420, 170, 500, 210)
rect_on_big = big_image.crop(box)
rect_on_big.save(PATH)
def download_dong(res): # 调取
input_ocr = browser.find_element(By.CSS_SELECTOR, "#codeValue")
input_ocr.send_keys(res)
time.sleep(1)
button = browser.find_element(By.CSS_SELECTOR, '.code_but input')
button.click()
def spider_way(num_max): # 操作浏览器,填写验证码点击提交
i = 0
while i < num_max:
ocr_img()
download_dong(recognize())
browser.refresh()
time.sleep(2)
i = i + 1
def run_t():
connect_web()
spider_way(NUM_MAX)
browser.close() # 关闭浏览器
if __name__ == '__main__':
t1 = threading.Thread(target=run_t(), name='t1')
t1.start()
后续可以用多线程操作