beautifulsoup实战用法
新手一枚,自学bs4顺便抓点好看的图嘿嘿嘿
`
from bs4 import BeautifulSoup
import requests
from urllib import request
import multiprocessing
import time
headers = {"User-Agent": 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/74.0.3729.169 Safa'}
url_list = []
url_list2 = []
a = 1
def Get_url(Url_Index):
data = requests.get(url=Url_Index, headers=headers)
soup = BeautifulSoup(data.content, 'lxml')
# print(soup)
img_s = soup.find_all('a', target='_blank')
for img in img_s:
url_list.append(img.get('href'))
def Get_pic(URL):
pic_list = []
data = requests.get(url=URL, headers=headers)
soup = BeautifulSoup(data.content, 'lxml')
# print(soup)
img_s = soup.find_all('img')
for img in img_s:
pic_list.append(img.get('src'))
# print(pic_list)
return pic_list
def Save_pic(url):
root = r'D:\爬虫总图\爬虫图片5/'
pic_path = root + str(a) + '.jpg'
request.urlretrieve(url=url, filename=pic_path)
print('第' + str(a) + '张图保存成功')
if __name__ == '__main__':
for i in range(1, 8):
url_index = 'https://www.meinvtu123.net/a/31/list_31_' + str(i) + '.html'
p1 = multiprocessing.Process(target=Get_url(Url_Index=url_index), args=(2,))
p1.start()
for j in url_list:
if j not in url_list2:
url_list2.append(j)
for x in range(0, len(url_list2)):
for y in range(0, len(Get_pic(URL=url_list2[x]))):
try:
list_url = Get_pic(URL=url_list2[x])[y]
print(list_url)
p3 = multiprocessing.Process(target=Save_pic(list_url), args=(4,)) # 多进程用法
p3.start()
except TimeoutError or ConnectionError as e:
continue
a += 1
time.sleep(0.8)
# print(url_list2)
`
爬取速度感觉还是太慢,但相较于selenium已经快很多了…
and这只是小网站,大厂的网站爬取属实有些困难,还需努力…