spider_review-爬虫复习

spider_review

import requests

session = requests.Session()
session.verify = False
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'
})
resp = session.get('https://movie.douban.com/top250')
print()
print()

正则表达式的应用

import re
import openpyxl
import requests
from openpyxl.cell import Cell
from openpyxl.styles import Font, Alignment

pattern = re.compile(r'<a\s.*?href="(?P<foo>.*?)".*?title="(?P<bar>.*?)".*?>')
resp = requests.get('https://www.sohu.com/')
# finditer方法会返回一个迭代器对象
iter_obj = pattern.finditer(resp.text)

# 创建一个工作簿
wb = openpyxl.Workbook()
# 获取第一个工作表
sheet = wb.active
sheet.title = '搜狐新闻'
# 修改表头单元格的宽度和高度
sheet.row_dimensions[1].height = 35
sheet.column_dimensions['A'].width = 80
sheet.column_dimensions['B'].width = 120
# 向单元格写入数据
sheet.cell(1, 1, '标题')
sheet.cell(1, 2, '链接')
# 修改指定单元格的样式
font = Font(size=18, name='华文楷体', bold=True, color='ff0000')
alignment = Alignment(horizontal='center', vertical='center')
for col_index in 'AB':
    curr_cell = sheet[f'{col_index}1']  # type: Cell
    curr_cell.font = font
    curr_cell.alignment = alignment
# 通过对迭代器对象的循环遍历可以获得Match对象
for index, matcher in enumerate(iter_obj):
    # # 获取整个a标签
    # print(matcher.group())
    # # 获取捕获组的内容
    # print(matcher.group('bar'))
    # print(matcher.group('foo'))
    sheet.cell(index + 2, 1, matcher.group('bar'))
    sheet.cell(index + 2, 2, matcher.group('foo'))

# 保存工作簿
wb.save('files/爬虫数据文件.xlsx')

# # 根据正则表达式匹配a标签的href和title属性得到一个列表
# anchors_list = pattern.findall(resp.text)
# for href, title in anchors_list:
#     print(href)
#     print(title)

破解selenium反爬最重要的一行代码

from selenium import webdriver
browser = webdriver.Chrome()

# 破解Selenium反爬最重要的一行代码
browser.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
    'source': 'Object.defineProperty(navigator, "webdriver", {get: () => undefined})'
})

browser.get('https://www.baidu.com')
browser.implicitly_wait(10)
anchor = browser.find_element_by_css_selector('#s-top-left > a:nth-child(7)')
# 通过WebElement对象的is_displayed方法判定元素是否可见
# 注意：不可见的超链接一般都不能访问，因为它极有可能是一个诱使爬虫访问的蜜罐链接
print(anchor.is_displayed())
print(anchor.size)
print(anchor.location)

# 浏览器窗口截图
browser.get_screenshot_as_file('files/baidu.png')

光学文字识别

import easyocr

reader = easyocr.Reader(['ch_sim', 'en'], gpu=False)
print(reader.readtext('files/idcard.jpg', detail=0))

从页面上抠图

from PIL import Image as img
from PIL import ImageFilter
from PIL.Image import Image

image = img.open('files/idcard.jpg')  # type: Image
print(image.size)
# 使用滤镜处理图片
emboss_image = image.filter(ImageFilter.EMBOSS)
emboss_image.show()
# 抠图
head = image.crop((320, 50, 460, 235))
# # 显示图片
# head.show()
# 保存图片
head.save('files/head.jpg')