第一章:爬虫基础简介
-
前戏:
- 你是否在夜深人静的时候,想看一些会让你更睡不着的图片
- 你是否在考试或者面试前夕,想看一些具有针对性的题目和面试题
- 你是否想在杂乱的网络世界获取你想要的数据
-
什么是爬虫:
- 通过编写程序,模拟浏览器上网,然后让其去互联网上抓取数据的过程
-
爬虫的价值:
- 实际应用
- 就业
第二章:requests模块
-
作用:
- 模拟浏览器发请求
-
如何使用:(requests模块的编码流程)
- 指定url
- 发起请求
- 获取相应数据
- 持久化存储
-
环境安装:
- pip install requests
第三章:验证码识别
-
肉眼识别(不推荐)
-
验证码识别平台(超级鹰、打码云……)
-
session对象
import requests from lxml import etree headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36' } #创建会话对象,该会话对象可以调用get和post发起请求 session = requests.Session() #使用会话对面对登录页面发起请求 page_text = session.get(url='https://github.com/login',headers=headers).text #解析出动态的taken值 tree = etree.HTML(page_text) t = tree.xpath('//*[@id="login"]/form/input[2]/@value')[0] #指定模拟登录请求的url url = 'https://github.com/session' #参数封装(处理动态taken值) data = { 'commit': 'Sign in', 'utf8': '✓', 'authenticity_token': t, 'login': 'bobo328410948@sina.com', 'password': 'bobo@15027900535', 'webauthn-support': 'supported', } #使用会话对象进行模拟登录请求发送(携带cookie) page_text = session.post(url=url,headers=headers,data=data).text #持久化存储 with open('./git.html','w',encoding='utf-8') as fp: fp.write(page_text)
第四章:代理IP
-
proxies参数处理代理IP:
import requests import random if __name__ == "__main__": #不同浏览器的UA header_list = [ # 遨游 {"user-agent": "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)"}, # 火狐 {"user-agent": "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"}, # 谷歌 { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"} ] #不同的代理IP proxy_list = [ {"http": "112.115.57.20:3128"}, {'http': '121.41.171.223:3128'} ] #随机获取UA和代理IP header = random.choice(header_list) proxy = random.choice(proxy_list) url = 'http://www.baidu.com/s?ie=UTF-8&wd=ip' #参数3:设置代理 response = requests.get(url=url,headers=header,proxies=proxy) response.encoding = 'utf-8' with open('daili.html', 'wb') as fp: fp.write(response.content)
第五章:高性能异步爬虫
-
进程池
#同步执行import timedef sayhello(str): print("Hello ",str) time.sleep(2) name_list =['xiaozi','aa','bb','cc'] start_time = time.time() for i in range(len(name_list)): sayhello(name_list[i]) print('%d second'% (time.time()-start_time))
执行结果
Hello xiaoziHello aaHello bbHello cc8 second
#异步基于线程池 import time from multiprocessing.dummy import Pool def sayhello(str): print("Hello ",str) time.sleep(2) start = time.time() name_list =['xiaozi','aa','bb','cc'] #实例化线程池对象,开启了4个线程 pool = Pool(4) pool.map(sayhello,name_list) pool.close() pool.join() end = time.time() print(end-start)
原则:线程池处理的是阻塞且耗时的操作
第六章:selenium模块
edge驱动
https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/
在浏览器中滚动页面
window.scrollTo(0,document.body.scrollHeight)
使用find_element()
from selenium.webdriver.common.by import By
driver.find_element_by_xpath('//*[@id="root"]/div/header/div[2]/aside/div[2]').click() #新用法
driver.find_element(By.XPATH,'//*[@id="pageBody"]/div/div[1]/div[1]/div[2]').click() #旧用法
selenium自动化代码
# -*-coding=utf-8-*-
# @Time:2022/5/712:01
# @Author:Autumn
# @File:02.selenium的其他自动化操作.py
# @Software:PyCharm
import requests
from lxml import etree
from selenium import webdriver
import time
from selenium.webdriver.common.by import By
if __name__ == '__main__':
bro=webdriver.Edge()
bro.get('https://www.taobao.com/')
#标签定位
search_input=bro.find_element(By.ID,'q')
#标签交互
search_input.send_keys('耳机')
#执行一组js程序
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
time.sleep(2)
#点击搜索按钮
#btn=bro.find_element_by_css_selector('.btn-search') #旧用法,不推荐使用
btn = bro.find_element(By.CSS_SELECTOR,'.btn-search')
btn.click()
time.sleep(2)
bro.get('https://www.baidu.com')
time.sleep(2)
bro.back() #后退
time.sleep(2)
bro.forward() #前进
time.sleep(2)
time.sleep(5)
#input('按任意键结束')
bro.quit()
无头浏览器
from selenium import webdriver
from time import sleep
# 实现无可视化界面
from selenium.webdriver.edge.options import Options
# 实现规避检测
from msedge.selenium_tools import EdgeOptions
from msedge.selenium_tools import Edge
# 实现无可视化界面的操作
EDGE = {
"browserName": "MicrosoftEdge",
"version": "",
"platform": "WINDOWS",
# 关键是下面这个
"ms:edgeOptions": {
'extensions': [],
'args': [
'--headless',
'--disable-gpu',
'--remote-debugging-port=9222',
]}
}
# 实现规避检测
option = EdgeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
# 如何实现让selenium规避被检测到的风险
bro = webdriver.Edge(capabilities=EDGE)
if __name__ == '__main__':
# 无可视化界面(无头浏览器)
bro.get("https://www.baidu.com")
from selenium import webdriver
from time import sleep
# 实现无可视化界面
from selenium.webdriver.edge.options import Options
# 实现规避检测
from msedge.selenium_tools import EdgeOptions
from msedge.selenium_tools import Edge
# 实现无可视化界面的操作
EDGE = {
"browserName": "MicrosoftEdge",
"version": "",
"platform": "WINDOWS",
# 关键是下面这个
"ms:edgeOptions": {
'extensions': [],
'args': [
'--headless',#不加的话就有头
'--disable-gpu',
'--remote-debugging-port=9222',
]}
}
# 实现规避检测
option = EdgeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
if __name__ == '__main__':
# 如何实现让selenium规避被检测到的风险
bro = webdriver.Edge(capabilities=EDGE)
#把msedgedriver.exe拷贝一份重命名为MicrosoftWebDriver.exe,并把它们两个拷贝到Python/Scripts安装目录下
#pip install msedge-selenium-tools
from selenium import webdriver
from time import sleep
# 实现无可视化界面
from selenium.webdriver.edge.options import Options
# 实现规避检测
from msedge.selenium_tools import EdgeOptions
from msedge.selenium_tools import Edge
# 实现无可视化界面的操作
EDGE = {
"browserName": "MicrosoftEdge",
"version": "",
"platform": "WINDOWS",
# 关键是下面这个
"ms:edgeOptions": {
'extensions': [],
'args': [
'--headless', #不加的话就有头
'--disable-gpu',
'--remote-debugging-port=9222',
]}
}
# 实现规避检测
option = EdgeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
# 如何实现让selenium规避被检测到的风险
bro = webdriver.Edge(executable_path='./MicrosoftWebDriver.exe', capabilities=EDGE)
# 无可视化界面(无头浏览器)
bro.get("https://www.baidu.com")
print(bro.page_source)
sleep(2)
bro.quit()
#把msedgedriver.exe拷贝一份重命名为MicrosoftWebDriver.exe,并把它们两个拷贝到Python安装目录下
#pip install msedge-selenium-tools
from selenium import webdriver
from time import sleep
#为了隐层浏览器导入的包(实现可视化界面)
from selenium.webdriver.edge.options import Options
#实现规避selenium检测
from msedge.selenium_tools import EdgeOptions
from msedge.selenium_tools import Edge
caps= {
"browserName": "MicrosoftEdge",
"version": "",
"platform": "WINDOWS",
# 关键是下面这个
"ms:edgeOptions": {
'extensions': [],
'args': [
'--headless',
'--disable-gpu',
'--remote-debugging-port=9222',
]}
}
# 实现规避检测
option = EdgeOptions()
option.add_experimental_option('excludeSwitches', ['enable-automation'])
# 如何实现让selenium规避被检测到的风险
bro=webdriver.Edge(executable_path='./msedgedriver.exe',capabilities=caps)
#无可视化(无头浏览器)
bro.get('https://www.baidu.com')
print(bro.page_source)
sleep(2)
bro.quit()
#把msedgedriver.exe拷贝一份重命名为MicrosoftWebDriver.exe,并把它们两个拷贝到Python安装目录下
#pip install msedge-selenium-tools
from msedge.selenium_tools import Edge, EdgeOptions
options = EdgeOptions()
options.use_chromium = True
options.add_argument('headless')
browser = Edge(options=options)
browser.get('https://www.baidu.com')
browser.quit()
pillow库的使用
#pip install pillow
from PIL import Image
import os
#修改文件名
#遍历文件夹下面的所有文件
for f in os.listdir('.'):
#如果文件为jpg格式
if f.endswith('.jpg'):
#则调用pillow将图片打开
i = Image.open(f)
#将文件分为文件路径和文件名
fn,ftext = os.path.splitext(f)
#以新的文件名保存图片(需要新建文件夹pngs)
i.save('pngs/{}.png'.format(fn))
#生成缩略图
#缩略图尺寸
size_300 = (300,300)
for f in os.listdir('.'):
if f.endswith('.jpg'):
i = Image.open(f)
fn,fext = os.path.splitext(f)
#生成缩略图(保持宽高比)
i.thumbnail(size_300)
#保存缩略图(需要新建文件夹300)
i.save('300/{}_300{}'.format(fn,fext))
#旋转图片
img = Image.open('./cat_3.jpg')
img_rotate90 = img.rotate(90)
img_rotate90.save('./cat_3_rotate90.jpg')
img_rotate90
#生成灰色图片
img = Image.open('./cat_3.jpg')
img_convertL = img.convert(mode='L')
img_convertL.save('./cat_3_convertL.jpg')
#添加高斯模糊
#引入滤镜类
from PIL import ImageFilter
#默认半径为2
img = Image.open('./cat_2.jpg')
img_filter_gaussian_default = img.filter(ImageFilter.GaussianBlur())
img_filter_gaussian_default
#半径为5
img_filter_gaussian_5 = img.filter(ImageFilter.GaussianBlur(5))
img_filter_gaussian_5
#半径为10
img_filter_gaussian_10 = img.filter(ImageFilter.GaussianBlur(5))
img_filter_gaussian_10
#屏幕截图
im1 = ImageGrab.grab()#不带参数表示全屏幕截图
im2 = ImageGrab.grab([0, 0, 200, 50])#截取屏幕指定区域的图像
im3 = ImageGrab.grab((0, 0, 200, 50))#截取屏幕指定区域的图像
im1.save('xxxxxx1.png')
im2.save('xxxxxx2.png')
im3.save('xxxxxx3.png')
#图像裁剪与粘贴
box = (120, 194, 220, 294) #定义裁剪区域
region = im.crop(box) #裁剪
region = region.transpose(Image.ROTATE_180)
im.paste(region, box) #粘贴
显示图片
使用matplotlib
import matplotlib.pyplot as plt # plt 用于显示图片
import matplotlib.image as mpimg # mpimg 用于读取图片
import numpy as np
lena = mpimg.imread('lena.png') # 读取和代码处于同一目录下的 lena.png
# 此时 lena 就已经是一个 np.array 了,可以对它进行任意处理
lena.shape #(512, 512, 3)
plt.imshow(lena) # 显示图片
plt.axis('off') # 不显示坐标轴
plt.show()
使用PIL
from PIL import Image
im = Image.open('lena.png') #打开
im.show() #显示
im.save('new_lena.png') #保存
进入iframe框架
#如果源码有iframe框架,必须先进入iframe框架
broswer.switch_to.frame(broswer.find_element(By.ID,''))
滑块验证码的识别
#pip install -U opencv-python
import cv2.cv2 as cv2 #这样写才能保证有提示
#读取背景图片的RGB码
bj_rgb=cv2.imread('filepath')
#颜色空间转换
img3 = cv2.cvtColor(img,cv2.COLOR_GRAY2RGB)
cv2.imshow('My honey',imgcopy)
cv2.waitKey(0)
## cv2.COLOR_X2Y,其中X,Y = RGB, BGR, GRAY, HSV, YCrCb, XYZ, Lab, Luv, HLS
#转化为灰度图
bj_gray = cv2.cvtColor(bj_img, cv2.COLOR_RGB2GRAY)
设置隐式等待
#设置隐式等待
broswer.implicitly_wait(5)
第七章:scrapy模块
在pycharm终端创建scrapy模板
cd 第五章:scrapy scrapy startproject FirstBlood cd FirstBlood scrapy genspider first www.xxx.com scrapy crawl first scrapy crawl first --nolog #bu'xing
附录一:
终端命令
#进入指定目录
cd path
#python包的备份与恢复
pip freeze > filepath\requirements.txt
sudo pip install -r filepath\requirements
常用代码
requests.get(url=url, headers=headers).content.decode("utf-8")
r.encoding = r.apparent_encoding #防止乱码
#通用处理中文乱码的解决方案
img_name=img_name.encode('iso-8859-1').decode('gbk')
import time
time.sleep(1) #休眠一秒
time.sleep(random.randint(0,5)) #随机休眠
response=requests.get(url=url,headers=headers,timeout=(3,7)) #设置连接时间3s,响应时间7s
while True:
try:
response=requests.get(url=url,headers=headers,timeout=5) #响应时间为5s
except:
prinit('error,rest and then restart')
time.sleep(5) #出错就休眠5s再发请求,再继续爬
else:
pass
# 准备好的代理ip
proxy = "127.0.0.1:8000"
proxies = {
"https": "https://{0}".format(proxy),
"http": "http://{0}".format(proxy),
}
res = requests.get(url, proxies=proxies)
list=tree.xpath('//div[@class="bottom"]/li[1]/a | //div[@id='photos']//img') #tree.xpath返回的是一个tree类的列表
for tag in list:
r=tag.xpath('./text()') #对tree再进行xpath,要从当前节点开始定位
#颜色输出
print("\033[31m这是红色字体\033[0m")
print("\033[32m这是绿色字体\033[0m")
print("\033[33m这是黄色字体\033[0m")
print("\033[34m这是蓝色字体\033[0m")
print("\033[38m这是默认字体\033[0m")
#删除文件
if os.path.exists(file_path):
# removing the file using the os.remove() method
os.remove(file_path)
#验证返回的是否是页面源码数据
print(response.status_code) #是200的话表示返回的是源码数据
#创建会话对象,该会话对象可以调用get和post发起请求
session = requests.Session()
#使用会话对面对登录页面发起请求
page_text = session.get(url='https://github.com/login',headers=headers).text
#css定位
.class #id tagname
#网页截图,将当前页面截取并保存
bro.save_screenshot('file_path')
#确定验证码对应的左上角和右下角对应的坐标
img_tag=bro.find_element(By)
bro.find_element(By.XPATH,'xpath路径')
#设置隐式等待
broswer.implicitly_wait(5)
相关模块的安装
pip install requests
pip install pillow #PIL图形处理库
pip install selenium
pip install lxml #xpath解析库
pip install pyinstaller
pip install easygui #GUI图形化界面
pip install pygame
pip install numpy
pip install matplotlib
pip i msedge.selenium_tools
pip install pyquery
相关模块的导入操作
from selenium import webdriver
from selenium.webdriver.common.by import By
from time import sleep
from PIL import Image
import os
import schedule
import re
import requests
from lxml import etree
import os
import json
import pygame
import matplotlib.pyplot as plt # plt 用于显示图片
import matplotlib.image as mpimg # mpimg 用于读取图片
import numpy as np
import tetesserocr
# 实现无可视化界面
from selenium.webdriver.edge.options import Options
# 实现规避检测
from msedge.selenium_tools import EdgeOptions
from msedge.selenium_tools import Edge
相关模块的使用
pyinstaller
#pip install pyinstaller
#在终端运行
pyinstaller -F/-D [-w/-c] [-i icopath] filepath
相关参数
headers={
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36 Edg/101.0.1210.39'
}
模板
#-*- coding = utf-8 -*-
#@Time : ${DATE} ${TIME}
#@Author : autumn
#@File : ${NAME}.py
#@Software : ${PRODUCT_NAME}
{
"HEADER":{
"prefix": "header",
"body": [
"#-*- coding = utf-8 -*-",
"#@Time : ${DATE} ${TIME}",
"#@Author : autumn",
"#@File : ${NAME}.py",
"#@Software : ${PRODUCT_NAME}"
],
}
}
{
"HEADER":{
"prefix": "header",
"body": [
"#!/usr/bin/python",
"# -*- encoding: utf-8 -*-",
"'''",
"@File : $TM_FILENAME",
"@Time : $CURRENT_YEAR/$CURRENT_MONTH/$CURRENT_DATE $CURRENT_HOUR:$CURRENT_MINUTE:$CURRENT_SECOND",
"@Author : Bing.Yao ",
"@Version : 1.0",
"@Contact : 123456@qq.com",
"@WebSite : https://blog.csdn.net/rbin_yao",
"'''",
"import logging",
"from logging.handlers import RotatingFileHandler",
"",
"def log_config():",
" LOG_FORMAT = '[%(asctime)s][%(levelname)s]: %(message)s'",
" level = logging.INFO",
" logging.basicConfig(level=level, format=LOG_FORMAT)",
" #创建RotatingFileHandler对象,满2MB为一个文件,共备份3个文件",
" log_file_handler = RotatingFileHandler(filename='test.log', maxBytes=2*1024*1024, backupCount=3)",
" # 设置日志打印格式",
" formatter = logging.Formatter(LOG_FORMAT)",
" log_file_handler.setFormatter(formatter)",
" logging.getLogger('').addHandler(log_file_handler)",
"",
"def main():",
" log_config()",
"",
"if __name__ == '__main__':",
" main()",
"",
"$0"
],
}
}
附录二:自制函数:
#获取响应,重连,编码,延迟
def url_get_sourse(url,type='text'):
url = url
while True:
try:
response = requests.get(url=url, headers=headers, timeout=(3, 7))
except:
print("\033[31m网络错误,5s后尝试重新连接\033[0m")
time.sleep(5)
else:
if type=="text":
response.encoding=response.apparent_encoding
page_text=response.text
response.encoding = response.apparent_encoding
page_text = response.text
return page_text
elif type=="json":
page_text=response.json()
return page_text
else:
page_text=response.content
return page_text
#改变ip
def change_ip():
global ipnum,proxies
ipnum+=1
proxy = ip_list[ipnum]
proxies = {
'http': 'http://' + proxy,
'https': 'https://' + proxy,
}
print(f'已将IP修改为{proxy}')
#获取ip列表
def get_ip_list():
filepath = '爬取数据/代理池/快代理.txt'
ip_list=[]
if os.path.exists(filepath):
with open(filepath, 'r') as fp:
list = fp.readlines()
# print(ist)
for ip in list:
ip = ip[:-2]
ip_list.append(ip)
return ip_list
后续:
- 深度学习
- 神经网络
- AI算法
- 前端
- 自动化办公
- GUI图形化界面
- 自动化测试
- 网络安全
- js
- potplayer高级应用
- 脚本的编写
- 按键精灵
- bat脚本
- ghost备份系统
- linux to go
- win to go
- 多系统
- UEFI与Legacy启动