一、爬取news
用selenium打开模拟界面
import time
from selenium import webdriver
driver = webdriver.Chrome('C:\\Users\dell\Python 爬虫 Study\chromedriver.exe')
driver.get('https://news.qq.com/')
driver.maximize_window()
#了解ajax加载
for i in range(1,100):
time.sleep(2)
driver.execute_script("window.scrollTo(window.scrollX, %d);"%(i*200))
用beautifulsoup爬取网页element。
from bs4 import BeautifulSoup
html=driver.page_source
bsObj=BeautifulSoup(html,"lxml")
bsObj
在Element中找到我们需要爬取的部分
jxtits=bsObj.find_all("div",{"class":"jx-tit"})[0].find_next_sibling().find_all("li")
jxtits
最后将爬取的信息输出
print("index",",","title",",","url")
for i,jxtit in enumerate(jxtits):
# print(jxtit)
try:
text=jxtit.find_all("img")[0]["alt"]
except:
text=jxtit.find_all("div",{"class":"lazyload-placeholder"})[0].text
try:
url=jxtit.find_all("a")[0]["href"]
except:
print(jxtit)
print(i+1,",",text,",",url)
二、进阶:ZH爬虫
import base64
import hashlib
import hmac
import json
import re
import threading
import time
from http import cookiejar
from urllib.parse import urlencode
import matplotlib
matplotlib.use('Qt5Agg')
import execjs
import requests
from PIL import Image
class ZhihuAccount(object):
"""
使用时请确定安装了(7.0 以上版本) 或其他 JS 环境
报错 execjs._exceptions.ProgramError: TypeError: 'exports' 就是没有安装
"""
def __init__(self, username: str = None, password: str = None):
self.username = username
self.password = password
self.login_data = {
'client_id': 'c3cef7c66a1843f8b3a9e6a1e3160e20',
'grant_type': 'password',
'source': 'com.zhihu.web',
'username': '',
'password': '',
'lang': 'en',
'ref_source': 'homepage',
'utm_source': ''
}
self.session = requests.session()
self.session.headers = {
'accept-encoding': 'gzip, deflate, br',
'Host': 'www.zhihu.com',
'Referer': 'https://www.zhihu.com/',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36'
}
self.session.cookies = cookiejar.LWPCookieJar(filename='./cookies.txt')
def login(self, captcha_lang: str = 'en', load_cookies: bool = True):
"""
模拟登录知乎
:param captcha_lang: 验证码类型 'en' or 'cn'
:param load_cookies: 是否读取上次保存的 Cookies
:return: bool
若在 PyCharm 下使用中文验证出现无法点击的问题,
需要在 Settings / Tools / Python Scientific / Show Plots in Toolwindow,取消勾选
"""
if load_cookies and self.load_cookies():
print('读取 Cookies 文件')
if self.check_login():
print('登录成功')
return True
print('Cookies 已过期')
self._check_user_pass()
self.login_data.update({
'username': self.username,
'password': self.password,
'lang': captcha_lang
})
timestamp = int(time.time() * 1000)
self.login_data.update({
'captcha': self._get_captcha(self.login_data['lang']),
'timestamp': timestamp,
'signature': self._get_signature(timestamp)
})
headers = self.session.headers.copy()
headers.update({
'content-type': 'application/x-www-form-urlencoded',
'x-zse-83': '3_1.1',
'x-xsrftoken': self._get_xsrf()
})
data = self._encrypt(self.login_data)
login_api = 'https://www.zhihu.com/api/v3/oauth/sign_in'
resp = self.session.post(login_api, data=data, headers=headers)
if 'error' in resp.text:
print(json.loads(resp.text)['error'])
if self.check_login():
print('登录成功')
return True
print('登录失败')
return False
def load_cookies(self):
"""
读取 Cookies 文件加载到 Session
:return: bool
"""
try:
self.session.cookies.load(ignore_discard=True)
return True
except FileNotFoundError:
return False
def check_login(self):
"""
检查登录状态,访问登录页面出现跳转则是已登录,
如登录成功保存当前 Cookies
:return: bool
"""
login_url = 'https://www.zhihu.com/signup'
resp = self.session.get(login_url, allow_redirects=False)
if resp.status_code == 302:
self.session.cookies.save()
return True
return False
def _get_xsrf(self):
"""
从登录页面获取 xsrf
:return: str
"""
self.session.get('https://www.zhihu.com/', allow_redirects=False)
for c in self.session.cookies:
if c.name == '_xsrf':
return c.value
raise AssertionError('获取 xsrf 失败')
def _get_captcha(self, lang: str):
"""
请求验证码的 API 接口,无论是否需要验证码都需要请求一次
如果需要验证码会返回图片的 base64 编码
根据 lang 参数匹配验证码,需要人工输入
:param lang: 返回验证码的语言(en/cn)
:return: 验证码的 POST 参数
"""
if lang == 'cn':
api = 'https://www.zhihu.com/api/v3/oauth/captcha?lang=cn'
else:
api = 'https://www.zhihu.com/api/v3/oauth/captcha?lang=en'
resp = self.session.get(api)
show_captcha = re.search(r'true', resp.text)
if show_captcha:
put_resp = self.session.put(api)
json_data = json.loads(put_resp.text)
img_base64 = json_data['img_base64'].replace(r'\n', '')
with open('./captcha.jpg', 'wb') as f:
f.write(base64.b64decode(img_base64))
img = Image.open('./captcha.jpg')
if lang == 'cn':
import matplotlib.pyplot as plt
plt.imshow(img)
print('点击所有倒立的汉字,在命令行中按回车提交')
points = plt.ginput(7)
capt = json.dumps({'img_size': [200, 44],
'input_points': [[i[0] / 2, i[1] / 2] for i in points]})
else:
img_thread = threading.Thread(target=img.show, daemon=True)
img_thread.start()
# 这里可自行集成验证码识别模块
capt = input('请输入图片里的验证码:')
# 这里必须先把参数 POST 验证码接口
self.session.post(api, data={'input_text': capt})
return capt
return ''
def _get_signature(self, timestamp: int or str):
"""
通过 Hmac 算法计算返回签名
实际是几个固定字符串加时间戳
:param timestamp: 时间戳
:return: 签名
"""
ha = hmac.new(b'd1b964811afb40118a12068ff74a12f4', digestmod=hashlib.sha1)
grant_type = self.login_data['grant_type']
client_id = self.login_data['client_id']
source = self.login_data['source']
ha.update(bytes((grant_type + client_id + source + str(timestamp)), 'utf-8'))
return ha.hexdigest()
def _check_user_pass(self):
"""
检查用户名和密码是否已输入,若无则手动输入
"""
if not self.username:
self.username = input('请输入手机号:')
if self.username.isdigit() and '+86' not in self.username:
self.username = '+86' + self.username
if not self.password:
self.password = input('请输入密码:')
@staticmethod
def _encrypt(form_data: dict):
with open('./encrypt.js') as f:
js = execjs.compile(f.read())
return js.call('Q', urlencode(form_data))
if __name__ == '__main__':
account = ZhihuAccount('', '')
account.login(captcha_lang='cn', load_cookies=False)
这里我没有爬取成功,以后等我能力更强了回来补上。