【爬虫】学习：反爬

CSS位置偏移反爬

python3网络爬虫开发实战第二版——7.7
GitHub

from selenium import webdriver
from pyquery import PyQuery as pq
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
import re


def parse_name(name_html):
    has_whole = name_html('.whole')
    if has_whole:
        return name_html.text()
    else:
        chars = name_html('.char')
        items = []
        for char in chars.items():
            items.append({
                'text': char.text().strip(),
                'left': int(re.search('(\d+)px', char.attr('style')).group(1))
            })
        items = sorted(items, key=lambda x: x['left'], reverse=False)
        return ''.join([item.get('text') for item in items])


browser = webdriver.Chrome()
browser.get('https://antispider3.scrape.center/')
WebDriverWait(browser, 10) \
    .until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '.item')))
html = browser.page_source
doc = pq(html)
names = doc('.item .name')

for name_html in names.items():
    name = parse_name(name_html)
    print(name)
browser.close()

Wonder
清白家风
法老的宠妃终结篇（上下册）
士为知己（全二册）
那些年，我们一起追的女孩
非我倾城（全三册）
明朝那些事儿
我和你的笑忘书
王小波全集第一卷
怦然心动
龙枪编年史（全3册）
龙枪传奇（全三册）
黎明之街
认知心理学及其启示
银河帝国2：基地与帝国
银河帝国：基地
小学教材全解-四年级语文下
越界言论（第3卷）

字体反爬

python3网络爬虫开发实战第二版——7.8
GitHub

score反爬
方法一：构建字体字典

from selenium import webdriver
from pyquery import PyQuery as pq
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
import re
import requests

url = 'https://antispider4.scrape.center/css/app.654ba59e.css'

response = requests.get(url,timeout=20)
pattern = re.compile('.icon-(.*?):before\{content:"(.*?)"\}')
results = re.findall(pattern, response.text)
icon_map = {item[0]: item[1] for item in results} # 构建字体字典


def parse_score(item):
    elements = item('.icon')
    icon_values = []
    for element in elements.items():
        class_name = (element.attr('class'))
        icon_key = re.search('icon-(\d+)', class_name).group(1)
        icon_value = icon_map.get(icon_key) # 查询字体字典
        icon_values.append(icon_value)
    return ''.join(icon_values)


browser = webdriver.Chrome()
browser.get('https://antispider4.scrape.center/')
WebDriverWait(browser, 10) \
    .until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '.item')))
    
html = browser.page_source
doc = pq(html)
items = doc('.item')
for item in items.items():
    name = item('.name').text()
    categories = [o.text() for o in item('.categories button').items()]
    score = parse_score(item)
    print(f'name: {name} categories: {categories} score: {score}')
browser.close()

name: 霸王别姬 - Farewell My Concubine categories: ['剧情', '爱情'] score: 9.5
name: 这个杀手不太冷 - Léon categories: ['剧情', '动作', '犯罪'] score: 9.5
name: 肖申克的救赎 - The Shawshank Redemption categories: ['剧情', '犯罪'] score: 9.5
name: 泰坦尼克号 - Titanic categories: ['剧情', '爱情', '灾难'] score: 9.5
name: 罗马假日 - Roman Holiday categories: ['剧情', '喜剧', '爱情'] score: 9.5
name: 唐伯虎点秋香 - Flirting Scholar categories: ['喜剧', '爱情', '古装'] score: 9.5
name: 乱世佳人 - Gone with the Wind categories: ['剧情', '爱情', '历史', '战争'] score: 9.5
name: 喜剧之王 - The King of Comedy categories: ['剧情', '喜剧', '爱情'] score: 9.5
name: 楚门的世界 - The Truman Show categories: ['剧情', '科幻'] score: 9.0
name: 狮子王 - The Lion King categories: ['动画', '歌舞', '冒险'] score: 9.0

方法二

from selenium import webdriver
from pyquery import PyQuery as pq
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait

browser = webdriver.Chrome()
browser.get('https://antispider4.scrape.center/')
WebDriverWait(browser, 10) \
    .until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '.item')))
html = browser.page_source
doc = pq(html)
items = doc('.item')
for item in items.items():
    name = item('.name').text()
    categories = [o.text() for o in item('.categories button').items()]
    score = item('.el-rate').attr('aria-valuenow') # '4.75'
    score=float(score)*2 # 9.5
    print(f'name: {name} categories: {categories} score: {score}')
browser.close()