python3入门级爬虫-淘女郎网站爬取

最新推荐文章于 2020-12-04 00:22:03 发布

清沐_1412

最新推荐文章于 2020-12-04 00:22:03 发布

阅读量1k

点赞数

分类专栏： python 文章标签： python 数据采集爬虫

本文链接：https://blog.csdn.net/DR_eamMer/article/details/79127500

版权

python 专栏收录该内容

6 篇文章 0 订阅

订阅专栏

爬虫实战计划的第一篇：淘女郎网站图片爬取

爬虫一般都是从爬取妹子图入门的，哈哈。这个算是最简单的爬虫了，只用了request库，也是最入门级别的爬虫方法。

ps:附上GitHub链接：淘女郎网站图片爬取

代码展示：

def open_url(url):          
    req = urllib.request.Request(url)
    req.add_header('User-Agent',
                   'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.3964.2 Safari/537.36')
    response = urllib.request.urlopen(req)
    html = response.read()
    return html

def get_MM_Data(url,currentPage=0):         #获取基本信息，以及user_id
    data={
    'q':'',
    'viewFlag':'A',
    'sortType':'default',
    'searchStyle':'',
    'searchRegion':'city:',
    'searchFansNum':'',
    'currentPage':currentPage,
    'pageSize':'100'
    }
    data=urllib.parse.urlencode(data)
    urll = urllib.request.Request(url, data=bytes(data, 'utf-8'))
    urll.add_header('user-agent',
                    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.3964.2 Safari/537.36')

    response = urllib.request.urlopen(urll)
    html = response.read().decode('gbk').encode('utf-8')
    htmlToDict=json.loads(html)
    return htmlToDict['data']['searchDOList']

def get_MM_ID(data):
    ID=[]
    for i in data:
        ID.append(i['userId'])
    return ID

def get_MM_Name(data):
    for i in data:
        user_name.append(i['realName'])
    return user_name
def get_MM_city(data):
    for i in data:
        user_city.append(i['city'])
    return user_city

def get_MM_Height(data):
    for i in data:
        user_height.append(i['height'])
    return user_height

def get_MM_Weight(data):
    for i in data:
        user_weight.append(i['weight'])
    return user_weight

def get_photos_url(url):
    album_id=[]
    html=urllib.request.Request(url)
    html.add_header('user-agent',
                    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.3964.2 Safari/537.36')
    html1=urllib.request.urlopen(html).read().decode('gbk')
    response=re.findall('album_id=[0-9]*',html1)
    for i in response:
        if i not in album_id:
            album_id.append(i)
    return album_id

def get_photos(url):
    pic_url=[]
    html = open_url(url)
    html1=html.decode('gbk')
    for i in json.loads(html1)['picList']:
        picUrl=i['picUrl']
        pic_url.append('https:'+picUrl)
    return pic_url

主函数：

if __name__=='__main__':

    url='https://mm.taobao.com/tstar/search/tstar_model.do?_input_charset=utf-8'
    data=get_MM_Data(url,1)

    if not os.path.exists('D://taoMM'): #判断路径是否存在
        os.mkdir('D://taoMM') #路径不存在，新建文件夹
        os.chdir('D://taoMM')

    ID=get_MM_ID(data)
    get_MM_city(data)
    get_MM_Name(data)
    get_MM_Weight(data)
    get_MM_Height(data)
    for j in range(0,ID.__len__()-1):
        folder_name = user_name[j] + '_' + user_city[j] + '_体重' + str(user_weight[j]) + '_身高' + str(user_height[j])
        if not os.path.exists('D:\\taoMM\\'+folder_name):
            os.mkdir("D:\\taoMM\\"+folder_name)

        album = get_photos_url(
            'https://mm.taobao.com/self/album/open_album_list.htm?_charset=utf-8&user_id%20={}'.format(ID[j]))
        print('当前模特：{}'.format(user_name[j]))
        print(album)
        photos = get_photos(
            'https://mm.taobao.com/album/json/get_album_photo_list.htm?user_id={0}&{1}&top_pic_id=0&cover=%2F%2Fimg.alicdn.com%2Fimgextra&page=1'.format(
                ID[j], album[0]))
        print(photos)
        for i in photos:
            name=i.split('/')[-1]

            with open('D://taoMM//'+folder_name+'/'+name,'wb') as f:
                try:
                    html=open_url(i)
                except urllib.error.HTTPError:
                    continue
                f.write(html)

效果展示：