爬虫初级实战

beautifulsoup实战用法

新手一枚，自学bs4顺便抓点好看的图嘿嘿嘿

`

from bs4 import BeautifulSoup
import requests
from urllib import request
import multiprocessing
import time

headers = {"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
                         '(KHTML, like Gecko) Chrome/74.0.3729.169 Safa'}

url_list = []
url_list2 = []
a = 1


def Get_url(Url_Index):
    data = requests.get(url=Url_Index, headers=headers)
    soup = BeautifulSoup(data.content, 'lxml')
    # print(soup)
    img_s = soup.find_all('a', target='_blank')
    for img in img_s:
        url_list.append(img.get('href'))


def Get_pic(URL):
    pic_list = []
    data = requests.get(url=URL, headers=headers)
    soup = BeautifulSoup(data.content, 'lxml')
    # print(soup)
    img_s = soup.find_all('img')
    for img in img_s:
        pic_list.append(img.get('src'))
    # print(pic_list)
    return pic_list


def Save_pic(url):
    root = r'D:\爬虫总图\爬虫图片5/'
    pic_path = root + str(a) + '.jpg'
    request.urlretrieve(url=url, filename=pic_path)
    print('第' + str(a) + '张图保存成功')


if __name__ == '__main__':
    for i in range(1, 8):
        url_index = 'https://www.meinvtu123.net/a/31/list_31_' + str(i) + '.html'
        p1 = multiprocessing.Process(target=Get_url(Url_Index=url_index), args=(2,))
        p1.start()
        for j in url_list:
            if j not in url_list2:
                url_list2.append(j)
        for x in range(0, len(url_list2)):
            for y in range(0, len(Get_pic(URL=url_list2[x]))):
                try:
                    list_url = Get_pic(URL=url_list2[x])[y]
                    print(list_url)
                    p3 = multiprocessing.Process(target=Save_pic(list_url), args=(4,))  # 多进程用法
                    p3.start()
                except TimeoutError or ConnectionError as e:
                    continue
                a += 1
                time.sleep(0.8)
    # print(url_list2)

`

爬取速度感觉还是太慢，但相较于selenium已经快很多了…
and这只是小网站，大厂的网站爬取属实有些困难，还需努力…