模拟浏览器整理电影榜单

c851666395

已于 2022-12-20 18:32:07 修改

阅读量212

点赞数

分类专栏：爬虫文章标签： python 爬虫开发语言

于 2022-12-20 11:25:07 首次发布

本文链接：https://blog.csdn.net/c851666395/article/details/128382075

版权

爬虫专栏收录该内容

3 篇文章 0 订阅

订阅专栏

（学习使用）

步骤

主要问题是滑块验证码和加密字体：

1、滑块验证主要用的是ddddocr包:

(1) 找到iframe并进入ifame

(2) 下载滑块图片和背景图片

(3) 将滑块图片和背景图片传入ddddocr 得到滑动距离

(4) 设置滑动路径

(5) 根据滑动路径滑动按钮

2、加密字体主要用的是TTFont和muggle_ocr包

(1) 根据源代码找到woff文件并下载

(2) 将woff文件分割保存为图片

(3) 使用muggle_ocr 识别保存的图片

(4) 根据识别的结果构建解密字典

(5) 爬取榜单并根据字典进行解密

导入模块

import urllib
import re
import os
import shutil
from lxml import etree
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
import time
import muggle_ocr

验证码

需要进入iframe

def into_iframe(wait):

    iframe = wait.until(
        EC.presence_of_all_elements_located((By.XPATH, '//iframe'))
    )

    # 验证码需要转到frame
    wait.until(
        EC.frame_to_be_available_and_switch_to_it(iframe[0])
    )

计算滑动距离

# 计算需要滑动的距离
import ddddocr


def use_ddddocr(left=26):
    det = ddddocr.DdddOcr(det=False, ocr=False)

    with open('./验证码/block.jpg', 'rb') as f:
        target_bytes = f.read()

    with open('./验证码/background.jpg', 'rb') as f:
        background_bytes = f.read()

    res = det.slide_match(target_bytes, background_bytes, simple_target=True)

    print(res)
    res['target_y'] += left  # 加上左边界

    distance = res['target'][0] / 2 - res['target_y']  # 渲染的图片和原始图片相比 差不多是0.5倍

    return distance

生成轨迹

# 轨迹
def get_track(distance):
    # 移动轨迹
    track = []
    # 当前位移
    current = 0

    # 减速阈值(到达某个距离后减速)
    mid = distance * 0.6

    # 计算间隔
    t = 0.2
    # 初速度
    v = 0
    # 循环 判断是否超出滑动距离
    while current < distance:
        if current < mid:
            # 加速度为正70
            a = 70  # 40
        else:
            # 加速度为负110
            a = -110

        # 初速度为v0
        v0 = v
        # 单前速度 v = v0 + at
        v = v0 + a * t
        # 移动距离 x = v0t + 0.5 * a * t^2 距离公式
        move = v0 * t + 0.5 * a * (t ** 2)
        # 滑块往回拉时也退出
        if move < 0:
            break
        # 当前位移
        current += move
        # 加入轨迹
        track.append(round(move))

    return track

移动滑块

from selenium.webdriver import ActionChains


# 移动滑块
def move_to_gap(slider, track):
    ActionChains(driver).click_and_hold(slider).perform()  # click_and_hold() 按住底部滑块

    for x in track:
        ActionChains(driver).move_by_offset(xoffset=x, yoffset=0).perform()

    time.sleep(0.5)

    # release() 松开鼠标
    ActionChains(driver).release().perform()

下载验证码图片并调用滑块接口

# 看是否有验证码

def verification(wait):

    # 背景图url
    slideBg = wait.until(EC.presence_of_element_located((
        By.XPATH, '//img[@id="slideBg"]')))

    slideBg_url = slideBg.get_attribute('src')

    # 滑块图url
    slideBlock = wait.until(EC.presence_of_element_located((
        By.XPATH, '//img[@id="slideBlock"]'
    )))

    slideBlock_url = slideBlock.get_attribute('src')

    # 左边界
    style = wait.until(EC.presence_of_element_located((
        By.XPATH, '//img[@id="slideBlock"]'
    ))).get_attribute('style').replace(' ', '').replace('px', '')[:-1]  # 最后一个;要去掉

    style_dict = {}

    for i in style.split(';'):
        splip_list = i.split(':')
        style_dict[splip_list[0]] = int(splip_list[1])

    left = style_dict['left']

    # 移动按钮
    slider = wait.until(EC.presence_of_element_located((
        By.XPATH, '//div[@class="tc-drag-thumb"]'
    )))

    # 下载图片

    os.mkdir('./验证码') if not os.path.exists('./验证码') else 1  # 创建文件夹

    urllib.request.urlretrieve(slideBg_url, './验证码/background.jpg')  # 下载背景图

    urllib.request.urlretrieve(slideBlock_url, './验证码/block.jpg')  # 下载滑块图

    distance = use_ddddocr(left)  # 获得需要滑动的距离
    print(distance)

    track = get_track(distance)  # 获得轨迹

    move_to_gap(slider, track)  # 移动滑块

    del_img = False  # 是否删除下载的图片
    if del_img:
        shutil.rmtree('./验证码')

加密字体

urllib response

def get_response(url, headers):
    request = urllib.request.Request(url, headers=headers)

    response = urllib.request.urlopen(request)

    return response


# 获得二进制的html
def get_html(url, headers):
    request = urllib.request.Request(url, headers=headers)

    response = urllib.request.urlopen(request)

    html = response.read()

    return html

下载woff文件

# 获得woff文件
def get_woff(html, headers):


    file_name = re.findall(r'//.+(/\w+\.woff)', html)[0][1:]  # woff路径
    print(file_name)

    # 下载字体的woff文件
    stone_url = "http://s3plus.meituan.net/v1/mss_73a511b8f91f43d0bdae92584ea6330b/font/" + file_name

    html = get_html(stone_url, headers)  # 使用urllib库访问woff网站

    os.mkdir('fonts') if not os.path.exists('fonts') else 1
    # 把字体文件写入本地
    with open('fonts/' + file_name, 'wb') as f:
        f.write(html)

    return file_name

woff文件分割保存为图片

# 字体拆分保存为图片工具

class ReportLabPen(BasePen):

    def __init__(self, glyphSet, path=None):
        BasePen.__init__(self, glyphSet)
        if path is None:
            path = Path()
        self.path = path

    def _moveTo(self, p):
        (x, y) = p
        self.path.moveTo(x, y)

    def _lineTo(self, p):
        (x, y) = p
        self.path.lineTo(x, y)

    def _curveToOne(self, p1, p2, p3):
        (x1, y1) = p1
        (x2, y2) = p2
        (x3, y3) = p3
        self.path.curveTo(x1, y1, x2, y2, x3, y3)

    def _closePath(self):
        self.path.closePath()


def woff2_image_service(font_name, image_path, fmt="png"):
    os.mkdir(image_path) if not os.path.exists(image_path) else 1

    # 获取字体
    font = TTFont(font_name)
    gs = font.getGlyphSet()
    glyph_names = font.getGlyphNames()

    print('glyph_names_len', glyph_names)
    for i in glyph_names:
        if i[0] == '.':  # 跳过'.notdef', '.null'
            continue
        g = gs[i]
        pen = ReportLabPen(gs, Path(fillColor=colors.red, strokeWidth=5))
        g.draw(pen)
        w, h = g.width, g.width * 2
        g = Group(pen.path)
        g.translate(0, 200)
        d = Drawing(w, h)
        d.add(g)
        image_file = image_path + "/" + i + ".png"
        renderPM.drawToFile(d, image_file, fmt)
        img = cv2.imread(image_file, -1)
        height, width = img.shape[:2]
        # 缩小图像
        size = (int(width * 0.05), int(height * 0.05))

        img = cv2.resize(img, size, cv2.INTER_AREA)
        cv2.imwrite(image_file, img)
        image_buffer = Image.open(image_file)
        mark_img = Image.new('RGBA', size, (255, 0, 0, 0))

        mark_img.paste(image_buffer, (0, 0))
        mark_img.save(image_file)

    return "OK"

muggle_ocr 识别字体

# 借助muggle_ocr 识别字体

def ocr_num(sdk, filename=r'./fonts/img'):
    imgs = os.listdir(filename)
    unicode_dict = {}
    for img_code in imgs:

        # 小写
        new_img_code = img_code.split('.')[0].lower()

        with open(f"./fonts/img/{img_code}", "rb") as f:
            img = f.read()
        text = sdk.predict(image_bytes=img)

        # 6比较容易识别成C
        if text == 'C' or text == 'c':
            text = '6'  # 之后用replace 要用str类型

        unicode_dict[new_img_code] = text

    return unicode_dict

获得加密前字符

# 获取加密前的字符
def get_data(html):
    # html_etree = etree.HTML(html.decode('utf-8')) # 用scrapy的没掌握
    html_etree = etree.HTML(html)  # urllib 要借用etree做xpath
    film_list = html_etree.xpath(r'//dl/dd')

    films = {}
    for film in film_list:
        film_detail = {}

        # film_detail['name'] = film.xpath(r'.//p[@class="name"]/text()').extract_first() scrapy用extract_first

        film_detail['name'] = film.xpath(r'.//p[@class="name"]/a/@title')[0]  # etree 返回的是数组

        film_detail['film_url'] = 'https://www.maoyan.com/' + film.xpath('./a/@href')[0]

        film_detail['star'] = film.xpath('.//p[@class="star"]/text()')[0]

        film_detail['releasetime'] = film.xpath('.//p[@class="releasetime"]/text()')[0]

        # 多标签的text 用 //text

        # 存在 \u 要处理
        film_detail['realtime_box_office'] = film.xpath('.//p[@class="realtime"]/span//text()')[0].encode(
            'raw_unicode_escape').decode('utf-8').replace('\\u', 'uni') + film.xpath('.//p[@class="realtime"]//text()')[2][0]

        film_detail['total_box_office'] = film.xpath('.//p[@class="total-boxoffice"]/span//text()')[0].encode(
            'raw_unicode_escape').decode('utf-8').replace('\\u', 'uni') + film.xpath('.//p[@class="total-boxoffice"]//text()')[2][0]


        # 不能直接replace \u
        # film_detail['realtime_box_office'] = film.xpath('.//p[@class="realtime"]/span//text()')[0] + film.xpath('.//p[@class="realtime"]//text()')[2][0]
        #
        # film_detail['total_box_office'] = film.xpath('.//p[@class="total-boxoffice"]/span//text()')[0] + film.xpath('.//p[@class="total-boxoffice"]//text()')[2][0]

        films[film_detail['name']] = film_detail  # 加入到字典

    return films

用字典配对解密

# 用ocr识别出数字代替unicode
def change_data(unicode_dict, films):
    for film_detail in films.values():  # 每一部电影
        for code in unicode_dict:  # 每一个数字编码
            film_detail['realtime_box_office'] = film_detail['realtime_box_office'].replace(code, unicode_dict[code]) # 实时票房
            film_detail['total_box_office'] = film_detail['total_box_office'].replace(code, unicode_dict[code]) # 总票房

    return films

保存至数据库

from pymongo import MongoClient
def save_mongdb(films):
    client = MongoClient(host='localhost', port=27017)
    # 库
    db = client['miaoyan']
    # 表
    collections = db['film']
    # 插入数据
    for film in films.values():
        collections.insert_one(film)

main

headers = {
    'User-Agent': '   ',
    "Cookie": "  "}

# 设置无头浏览器
option = webdriver.ChromeOptions()
# option.add_argument('--headless')

# 隐藏特征(UA)
option.add_argument('user-agent=   ')
driver = webdriver.Chrome(options=option)
# 隐藏特征(js)
with open('stealth.min.js') as f:
    js = f.read()
# 执行js
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
  "source": js
})


# 测试验证码
  url = '猫眼电影的网站'

wait = WebDriverWait(driver, 10)

driver.get(url)

time.sleep(2) # 等待页面加载

try: # 看是否出现验证码
    into_iframe(wait)
    verification(wait)
    while True:
        time.sleep(3) # 等待页面跳转或刷新
        iframe = driver.find_element(by=By.XPATH, value='//iframe')
        if iframe is None: # 也许一次验证不成功
            break
        else:
            verification(wait) # 以后可以加个判断是二次验证还是验证失败

except:  # 未出现验证码
    print('未出现验证码')

# 退回父级主页面 不过这里是跳转
# driver.switch_to.parent_frame()


# time.sleep(3) 上面已经休眠过了

html = driver.page_source # 获得页面源码


# 原始的票房数字编码
films = get_data(html)

print(films)


# 下载woff 文件
file_name = get_woff(html=html, headers=headers)

image_dir = r"./fonts/img"  # 路径中不能出现中文

file_path = r'./fonts/' + file_name

# woff保存成图片
woff2_image_service(file_path, image_dir)

# 初始化sdk；model_type 包含了 ModelType.OCR/ModelType.Captcha 两种模式,分别对应常规图片与验证码
sdk = muggle_ocr.SDK(model_type=muggle_ocr.ModelType.OCR)
unicode_dict = ocr_num(sdk, filename=r'./fonts/img')

print('----字典表----')
print(unicode_dict)

print('----解码后----')
films = change_data(unicode_dict, films)
print(films)

# 保存至数据库
save_mongdb(films)

# 是否删除解码的图片和woff
del_img = True
del_woff = True

if del_img:
    shutil.rmtree(image_dir)

if del_woff:
    os.remove(file_path)