再爬mm图片

再爬mm图片

今天在社区看到一个同学写的mm图片爬虫,就很仔细的读了全部代码,正好,我也在学习爬虫,就以我目前的理解写了一遍.发出来做一交流:

首先我用的是requests的异步库发起的网络请求.

from requests_html import AsyncHTMLSession
from headers1 import *
import os

其次,让我觉得requests_html库自带的解析方法,很省心.

但我们今天爬取的不是一个动态链接,相对容易一些!下面就贴出全部代码:

from requests_html import AsyncHTMLSession
from headers1 import *
import os

url_t = 'https://pic.netbian.com/4kmeinv/index_{}.html'
asession = AsyncHTMLSession()
save_img = '屏图'
if save_img not in os.listdir('./'):
    os.mkdir(save_img)


async def main():
    for page in range(10):
        if page == 1:
            url = 'https://pic.netbian.com/4kmeinv/'
        else:
            url = url_t.format(str(page))
        try:
            with await asession.get(url=url, headers=header) as resp:
                print(resp.status_code, resp.encoding)
                html_r = resp.html.find('.slist .clearfix')
                for i in html_r:
                    each_url = list(
                        map(lambda x: x.attrs['href'], i.find('li a')))
                    each_title = list(map(lambda x: x.text, i.find('li a')))
                    tasks = dict(zip(each_url, each_title))
                    print(tasks)
                    await get_img_handle(asession, tasks)
        except Exception as e:
            print(e)


async def get_img_handle(asession, tasks):

    try:
        for url, title in tasks.items():
            src_link = 'https://pic.netbian.com' + url
            with await asession.get(url=src_link, headers=header) as resp:
                img_data = 'https://pic.netbian.com' + resp.html.find(
                    '.photo-pic #img img', first=True).attrs['src']
            img_name = title + '.jpg'
            save_img_dir = os.path.join(save_img, img_name)
            with await asession.get(url=img_data, headers=header) as resp:
                src_data = resp.content
                with open(save_img_dir, 'ab') as fp:
                    fp.write(src_data)
    except Exception as e:
        print(e)


asession.run(main)

最后,也上个图: