一个简单的爬虫
步骤
- 抓取页面
- 解析页面
- 抽取数据
- 存储数据
数据库的使用
- redis
#找到需要的内容和链接
client = redis.Redis(host='47.106.171.59', password=123456,port='6379')
# client.set(key,value)
# client.hset(kind,key,vlaue)
- myssql
conn = pymysql.connect(host='localhost', port=3306,
database='crawler', user='root',
password='123456', charset='utf8')
cursor = conn.cursor()
sql = ''
cursor.executemany(sql)
conn.commit()
cursor.close()
conn.close()
- mongodb
client = pymongo.MongoClient(host='47.106.171.59', port='27017')
db = client.db_name
pages = db.tb_name
# pages.insert_one({key,value})
pages.insert_many({},{},{})
抓取页面
- 抓取静态页面
# 通过指定的字符集对页面进行解码(不是每个网站都将字符集设置为utf-8)
def decode_page(page_bytes, charsets=('utf-8',)):
page_html = None
for charset in charsets:
try:
page_html = page_bytes.decode(charset)
break
except UnicodeDecodeError:
pass
# logging.error('Decode:', error)
return page_html
# 获取页面的HTML代码(通过递归实现指定次数的重试操作)
def get_page_html(seed_url, *, retry_times=3, charsets=('utf-8',)):
page_html = None
try:
page_html = decode_page(urlopen(seed_url).read(), charsets)
except URLError:
# logging.error('URL:', error)
if retry_times > 0:
return get_page_html(seed_url, retry_times=retry_times - 1,
charsets=charsets)
return page_html
- 抓取动态页面
webdriver官方文档
driver = webdriver.Chrome()
driver.get('https://v.taobao.com/v/content/live?spm=a21xh.11312891.servList.1.1ae97001cn78J4&catetype=701')
elem = driver.find_element_by_css_selector('input[placeholder="输入关键词搜索"]')
elem.send_keys('美女')
elem.send_keys(Keys.ENTER)
soup = BeautifulSoup(driver.page_source, 'lxml')
解析页面
- 主要是获取并筛选出页面里的链接,然后对链接进行处理,变成完整的链接,然后存放到任务列表里。
# 解析页面
def parse(self, html_page, *, domain='m.sohu.com'):
soup = BeautifulSoup(html_page, 'lxml') # 创建一个树结构
# 用urlparse 处理链接,变成正规的url
for a_tag in soup.body.select('a[href]'):
parser = urlparse(a_tag.attrs['href'])
scheme = parser.scheme or 'http'
netloc = parser.netloc or domain
if netloc != 'javascript' and netloc == domain:
path = parser.path
query = '?' + parser.query if parser.query else ''
full_url = f'{scheme}://{netloc}{path}{query}'
if not redis_client.sismember('visited_urls', full_url):
redis_client.rpush('m_sohu_task', full_url)
抽取数据
- content,主要是找出这个页面里需要的内容,然后以一定的方式存放,准备放到数据库里,持久化处理做准备
# 抽取数据,标题和内容P
def extract(self, html_page):
soup = BeautifulSoup(html_page, 'lxml')
title = soup.find('div', id='p_left').h2.get_text()
p_content = soup.find('div', id='p_left').find_all('p')
p_set = []
for p in p_content:
if p.get_text() not in ['', ' ', '\xa0']:
p_set.append(p.get_text())
my_tuple = (title, p_set)
return my_tuple
持久化数据
- 将需要的数据,放到数据库里,一般来说,爬取的url放到redis里,一个作为待任务列表,一个作为已完成任务列表。然后把爬取的大量内容放到mongo里,分类存储。对于爬出的精华,可以放到关系型数据库,mysql里。
mongo_client = pymongo.MongoClient(host='47.106.171.59', port=27017)
db = mongo_client.geyan
geyan_data_coll = db.content
if html_page not in [None, '']:
# 加密,存储
my_tuple = self.spider.extract(html_page)
my_dict = {
'title': my_tuple[0],
'page': my_tuple[1]
}
geyan_data_coll.insert_one(my_dict)
如何跳过表单
- 常规方法
import requests
from bs4 import BeautifulSoup
def main():
"""
常规方法跳过表单,获取页面
:return:
"""
resp = requests.get('https://github.com/login')
if resp.status_code != 200:
print('页面拿不到')
# return
cookies = resp.cookies.get_dict()
soup = BeautifulSoup(resp.text, 'lxml') # lxml 解析器, 效果好一点
utf8_value = soup.select_one('form input[name=utf8]').attrs['value']
authenticity_token = soup.select_one('form input[name=authenticity_token]').attrs['value']
data = {
'utf8': utf8_value,
'authenticity_token': authenticity_token,
'login': 'zhangli9479@163.com',
'password': 'Zl1994-7-9'
}
# 上传文件
# files = {
# 'files1': open(),
# 'files2': open()
# }
resp2 = requests.post('https://github.com/login', data=data, cookies=cookies)
print(resp2.content.decode('utf-8'))
if __name__ == '__main__':
main()
2.简便方法
import requests
import robobrowser
# 利用robobrowser 第三方工具来登录表单。
def main():
b = robobrowser.RoboBrowser(parser='lxml')
b.open('https://github.com/login')
f = b.get_form(action='/session')
f['login'].value = 'zhangli9479@163.com'
f['password'].value = 'Zl1994-7-9'
b.submit_form(f)
# print(b)
for a_tag in b.select('a[href]'):
print(a_tag.attrs['href'])
if __name__ == '__main__':
main()
生成验证码 及 验证码识别
- 生成验证码,
- 验证码识别
# pillow的使用, open save. 廖雪峰
import base64
import requests
from PIL import Image, ImageFilter
from pygments import BytesIO
from pytesseract import image_to_string
def main():
gui_do_img = Image.open(open('a2.jpg', 'rb'))
gui_do_img1 = gui_do_img.filter(ImageFilter.GaussianBlur)
gui_do_img1.save(open('a2a.jpg', 'wb'))
img1 = Image.open(open('a7.jpg', 'rb'))
img2 = img1.point(lambda x: 0 if x < 128 else 255)
img2.save(open('a7a.jpg', 'wb'))
print(image_to_string(img2))
resp = requests.get(
'https://pin2.aliyun.com/get_img?type=150_40&identity=mailsso.mxhichina.com&sessionid=k0xHyBxU3K3dGXb59mP9cdeTXxL9gLHSTKhRZCryHxpOoyk4lAVuJhgw==')
img3 = Image.open(BytesIO(resp.content))
img3.save('hello.jpg')
print(image_to_string(img3))
print(base64.b64encode(resp.content))
if __name__ == '__main__':
main()