# coding:utf-8 ''' 使用requests模块,使用代理,进行淘宝网商品信息的爬取 ''' # 引入需要的模块 import requests import random import re from lxml import etree # 定义免费代理列表 proxy_list = [{"http": "116.8.83.3:8118"}, {"http": "116.8.83.3:8118"}, {"http": "113.89.59.161:8118"}, {"http": "113.67.183.196:8118"}, {"http": "180.155.135.224:31425"}, {"http": "123.161.153.238:22593"}] # 定义访问路由 url = 'https://uland.taobao.com/sem/tbsearch?refpid=mm_26632360_8858797_29866178&keyword=%E5%A5%B3%E8%A3%85&clk1=51ae9dffa6be64388c59f2c5c20274c7&upsid=51ae9dffa6be64388c59f2c5c20274c7' # 设置请求头 headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36' } # 发送请求 response = requests.get(url, headers=headers, proxies=random.choice(proxy_list), verify=False) content = response.text # 图片名称的获取 reg_name = r'<span class="title" title=".*">(.*?)</span>' pattern_name = re.compile(reg_name) titles_list = pattern_name.findall(content) titles = [] for title in titles_list: titles.append(title) print "商品名称爬取结束" # 图片的获取 reg_name = r'<img data-ks-lazyload="(.*?)" s' pattern_name = re.compile(reg_name) imgs_list = pattern_name.findall(content) imgs = [] # 修改连接 for i in range(0, len(imgs_list) - 5): imgs_list[i] = "https:" + imgs_list[i] # 获取图片的url,进行第二次爬取 i = 1 for img in imgs_list: filename = str(i) + '_' + img[-20:] f = open('tb_img/%s' % filename, 'wb') url = img response = requests.get(url, verify=False) f.write(response.content) f.close() i += 1 print "图片爬取结束"
Python爬虫——4.6使用requests和正则表达式、随机代理爬取淘宝网商品信息
最新推荐文章于 2024-07-18 20:15:36 发布