spider_study2

一个简单的爬虫

步骤

1. 抓取页面
2. 解析页面
3. 抽取数据
4. 存储数据

数据库的使用

• redis

 #找到需要的内容和链接
    client = redis.Redis(host='47.106.171.59', password=123456,port='6379')
    # client.set(key,value)
    # client.hset(kind,key,vlaue)

• myssql

conn = pymysql.connect(host='localhost', port=3306,
                           database='crawler', user='root',
                           password='123456', charset='utf8')
    cursor = conn.cursor()
    sql = ''
    cursor.executemany(sql)
    conn.commit()
    cursor.close()
    conn.close()

• mongodb

client = pymongo.MongoClient(host='47.106.171.59', port='27017')
    db = client.db_name
    pages = db.tb_name
    # pages.insert_one({key,value})
    pages.insert_many({},{},{})

• 菜鸟教程

抓取页面

1. 抓取静态页面

# 通过指定的字符集对页面进行解码(不是每个网站都将字符集设置为utf-8)
def decode_page(page_bytes, charsets=('utf-8',)):
    page_html = None
    for charset in charsets:
        try:
            page_html = page_bytes.decode(charset)
            break
        except UnicodeDecodeError:
            pass
            # logging.error('Decode:', error)
    return page_html

# 获取页面的HTML代码(通过递归实现指定次数的重试操作)
def get_page_html(seed_url, *, retry_times=3, charsets=('utf-8',)):
    page_html = None
    try:
        page_html = decode_page(urlopen(seed_url).read(), charsets)
    except URLError:
        # logging.error('URL:', error)
        if retry_times > 0:
            return get_page_html(seed_url, retry_times=retry_times - 1,
                                 charsets=charsets)
    return page_html

1. 抓取动态页面
   webdriver官方文档

driver = webdriver.Chrome()
    driver.get('https://v.taobao.com/v/content/live?spm=a21xh.11312891.servList.1.1ae97001cn78J4&catetype=701')
    elem = driver.find_element_by_css_selector('input[placeholder="输入关键词搜索"]')
    elem.send_keys('美女')
    elem.send_keys(Keys.ENTER)
    soup = BeautifulSoup(driver.page_source, 'lxml')

解析页面

1. 主要是获取并筛选出页面里的链接，然后对链接进行处理，变成完整的链接，然后存放到任务列表里。

# 解析页面
    def parse(self, html_page, *, domain='m.sohu.com'):
        soup = BeautifulSoup(html_page, 'lxml')  # 创建一个树结构
        # 用urlparse 处理链接，变成正规的url
        for a_tag in soup.body.select('a[href]'):
            parser = urlparse(a_tag.attrs['href'])
            scheme = parser.scheme or 'http'
            netloc = parser.netloc or domain
            if netloc != 'javascript' and netloc == domain:
                path = parser.path
                query = '?' + parser.query if parser.query else ''
                full_url = f'{scheme}://{netloc}{path}{query}'
                if not redis_client.sismember('visited_urls', full_url):
                    redis_client.rpush('m_sohu_task', full_url)

抽取数据

1. content，主要是找出这个页面里需要的内容，然后以一定的方式存放，准备放到数据库里，持久化处理做准备

# 抽取数据,标题和内容P
    def extract(self, html_page):
        soup = BeautifulSoup(html_page, 'lxml')
        title = soup.find('div', id='p_left').h2.get_text()
        p_content = soup.find('div', id='p_left').find_all('p')
        p_set = []
        for p in p_content:
            if p.get_text() not in ['', ' ', '\xa0']:
                p_set.append(p.get_text())
        my_tuple = (title, p_set)
        return my_tuple

持久化数据

1. 将需要的数据，放到数据库里，一般来说，爬取的url放到redis里，一个作为待任务列表，一个作为已完成任务列表。然后把爬取的大量内容放到mongo里，分类存储。对于爬出的精华，可以放到关系型数据库，mysql里。

mongo_client = pymongo.MongoClient(host='47.106.171.59', port=27017)
db = mongo_client.geyan
geyan_data_coll = db.content
                if html_page not in [None, '']:
                    # 加密，存储
                    my_tuple = self.spider.extract(html_page)
                    my_dict = {
                        'title': my_tuple[0],
                        'page': my_tuple[1]
                    }
                    geyan_data_coll.insert_one(my_dict)

如何跳过表单

1. 常规方法

import requests
from bs4 import BeautifulSoup


def main():
    """
    常规方法跳过表单，获取页面
    :return:
    """
    resp = requests.get('https://github.com/login')
    if resp.status_code != 200:
        print('页面拿不到')
        # return
    cookies = resp.cookies.get_dict()
    soup = BeautifulSoup(resp.text, 'lxml')  # lxml 解析器， 效果好一点
    utf8_value = soup.select_one('form input[name=utf8]').attrs['value']
    authenticity_token = soup.select_one('form input[name=authenticity_token]').attrs['value']
    data = {
        'utf8': utf8_value,
        'authenticity_token': authenticity_token,
        'login': 'zhangli9479@163.com',
        'password': 'Zl1994-7-9'
    }
    # 上传文件
    # files = {
    #     'files1': open(),
    #     'files2': open()
    # }
    resp2 = requests.post('https://github.com/login', data=data, cookies=cookies)
    print(resp2.content.decode('utf-8'))


if __name__ == '__main__':
    main()

2.简便方法

import requests
import robobrowser


# 利用robobrowser 第三方工具来登录表单。
def main():
    b = robobrowser.RoboBrowser(parser='lxml')
    b.open('https://github.com/login')
    f = b.get_form(action='/session')
    f['login'].value = 'zhangli9479@163.com'
    f['password'].value = 'Zl1994-7-9'
    b.submit_form(f)
    # print(b)
    for a_tag in b.select('a[href]'):
        print(a_tag.attrs['href'])


if __name__ == '__main__':
    main()

生成验证码 及 验证码识别

1. 生成验证码，
2. 验证码识别

# pillow的使用,  open save. 廖雪峰
import base64

import requests
from PIL import Image, ImageFilter
from pygments import BytesIO
from pytesseract import image_to_string


def main():
    gui_do_img = Image.open(open('a2.jpg', 'rb'))
    gui_do_img1 = gui_do_img.filter(ImageFilter.GaussianBlur)
    gui_do_img1.save(open('a2a.jpg', 'wb'))

    img1 = Image.open(open('a7.jpg', 'rb'))
    img2 = img1.point(lambda x: 0 if x < 128 else 255)
    img2.save(open('a7a.jpg', 'wb'))

    print(image_to_string(img2))

    resp = requests.get(
        'https://pin2.aliyun.com/get_img?type=150_40&identity=mailsso.mxhichina.com&sessionid=k0xHyBxU3K3dGXb59mP9cdeTXxL9gLHSTKhRZCryHxpOoyk4lAVuJhgw==')
    img3 = Image.open(BytesIO(resp.content))
    img3.save('hello.jpg')
    print(image_to_string(img3))
    print(base64.b64encode(resp.content))


if __name__ == '__main__':
    main()