爬取（明星网）明星面部数据-CSDN博客

本文链接：https://blog.csdn.net/zhaoyin214/article/details/89085348

爬取（明星网）明星面部数据

from bs4 import BeautifulSoup
import os
import requests
import time

1 下载数据

1.1 请求分析

Request

GET /upload/thumb/2015/11-16/0-uwo1Wk.jpg HTTP/1.1
Host: img.mingxing.com
Referer:http://img.mingxing.com//mingxing//20181015/88aa35c304dc06e822bb2efdd33497a5.jpg
User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6821.400 QQBrowser/10.3.3040.400

def get_img(url, path):
    
    headers={
   
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6821.400 QQBrowser/10.3.3040.400",
        "Referer": url,
    }

    response = requests.get(url, headers=headers)
    # print(response.content)
    
    with open(path, "wb") as fw:
        fw.write(response.content)

if __name__ == "__main__":
    url = "http://img.mingxing.com//mingxing//20181015/88aa35c304dc06e822bb2efdd33497a5.jpg"
    path = "./dataset/tmp.jpg"
    
    get_img(url, path)

2 明星列表页面

Request

GET /ziliao/index?&p=1 HTTP/1.1
Host: www.mingxing.com
Connection: keep-alive
User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6821.400 QQBrowser/10.3.3040.400
Upgrade-Insecure-Requests: 1
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8
Accept-Encoding: gzip, deflate
Accept-Language: zh-CN,zh;q=0.9
Cookie: __51cke__=; UM_distinctid=169c7147623b32-041b6f54ccddbe-3257487f-232800-169c7147624ddd; CNZZDATA30054349=cnzz_eid%3D515205138-1553821270-https%253A%252F%252Fwww.baidu.com%252F%26ntime%3D1553821270; PHPSESSID=9btdnv30htpj54ies9em19pan1; right_adv=%7B%22time%22%3A%222019329%22%2C%22number%22%3A20%7D; __tins__18838395=%7B%22sid%22%3A%201553825001869%2C%20%22vd%22%3A%203%2C%20%22expires%22%3A%201553827085186%7D; __51laig__=21

2.1 单页明星列表

URL_MINGXING_CELEBRITY_LIST = "http://www.mingxing.com/ziliao/index"

<div>：class="page_starlist"，明星列表
-><ul>
--><li>
---><a>：明星页面url
----><span>
-----><img>：src - 明星图片url，alt = 明星姓名

def get_celebrities_one_page(url, idx_page):
    
    headers={
   
        "Connection": "keep-alive",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6821.400 QQBrowser/10.3.3040.400",
        "Upgrade-Insecure-Requests": "1",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",