任务1:利用cookie可以免去登录的烦恼(验证码)
'''
只需要有登录后的cookie,就可以绕过验证码
登录后的cookie可以通过Selenium用第三方(微博)进行登录,不需要进行淘宝的滑动验证码
'''
import requests
from urllib.parse import urlencode
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36',
# 登录后的cookie
'cookie': 'xxx',
}
params = {
'q': 'iphone',
'imgfile':'',
'commend': 'all',
'ssid': 's5-e',
'search_type': 'item',
'sourceId': 'tb.index',
'spm': 'a21bo.2017.201856-taobao-item.2',
'ie': 'utf8',
'initiative_id': 'tbindexz_20170306',
}
url = 'https://s.taobao.com/search?' + urlencode(params)
s = requests.Session()
response = s.get(url,headers=headers,verify=False).text
print(response)
任务2:爬取淘宝商品信息
from selenium import webdriver
# 通用选择
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ActionChains
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# 超时
from selenium.common.exceptions import TimeoutException
from lxml import etree
import json
import random
import time
browser = webdriver.Chrome()
browser.set_window_size(1400, 900)
'''
爬到第19页的时候淘宝会弹窗,熟悉的滑动解锁= =,以后解决了这个滑动在继续吧
总体思路没问题
'''
def taobao_login():
'''
淘宝的滑动验证码过不去,使用第三方登录
'''
'''
# 需要先登录
login = browser.find_element(By.ID,'J_Quick2Static').click()
username = browser.find_element(By.CSS_SELECTOR,'#TPL_username_1')
username.send_keys('XXX')
password = browser.find_element(By.CSS_SELECTOR,'#TPL_password_1')
password.send_keys('XXX')
button = browser.find_element(By.ID, 'J_SubmitStatic').click()
ActionChains(browser).move_by_offset(random.randint(10, 60), random.randint(10, 60)).perform()
# 判断大小
input2 = browser.find_element(By.ID,'nc_1__scale_text')
print(input2.size)
# 滑动验证码
action = ActionChains(browser)
source = browser.find_element(By.ID,'nc_1_n1z')
# 按住不放
action.click_and_hold(source).perform()
# 需要滑动的坐标
action.move_by_offset(298,0)
# 释放鼠标
action.release().perform()
'''
def weibo_login():
'''
账号密码输入后有滑动验证码,滑动成功也无法登录,考虑绕过去,采用第三方登录
通过微博账号登录
'''
weibo_button = browser.find_element(By.CSS_SELECTOR, '.weibo-login')
weibo_button.click()
# 网速有点慢 需要先注册一个微博账号且绑定一个淘宝账号,真麻烦= =
# 多登录几次,微博就会弹出当前网络超时,请稍后再试(600002),需要等一段时间才能进去
time.sleep(2)
username = browser.find_element(By.NAME, 'username')
username.send_keys('账号')
time.sleep(1)
password = browser.find_element(By.NAME, 'password')
password.send_keys('密码')
browser.find_element(By.CSS_SELECTOR, '.W_btn_g').click()
def index_page(page):
print('正在抓取第', page, '页')
try:
if page > 1:
# 等待直到页码输入框出现
input_box = wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager div.form > input')))
# 等待确定按钮可以被点击
submit = wait.until(
EC.element_to_be_clickable((By.CSS_SELECTOR, '#mainsrp-pager div.form > span.btn.J_Submit')))
input_box.clear()
input_box.send_keys(page)
submit.click()
# time.sleep(2)
# 等待直到跳转的页码等于高亮显示的页码,说明跳转成功
wait.until(
EC.text_to_be_present_in_element((By.CSS_SELECTOR, '#mainsrp-pager li.item.active > span'), str(page)))
# 等待直到所有商品加载出来
wait.until(
EC.presence_of_element_located((By.CSS_SELECTOR, '.m-itemlist .items .item')))
# 爬取详细信息
# time.sleep(3)
get_products()
except TimeoutException:
# 超时就重试一次
index_page(page)
def get_products():
html = etree.HTML(browser.page_source)
items = html.xpath('//div[@class="m-itemlist"]//div[@class="items"]/div')
for item in items:
product = {}
product['image'] = item.xpath('.//img/@data-src')
product['price'] = item.xpath('.//strong/text()')
product['title'] = item.xpath('.//img[@class="J_ItemPic img"]/@alt')
product['shop'] = item.xpath('.//a[@class="shopname J_MouseEneterLeave J_ShopInfo"]/span[2]/text()')
product['location'] = item.xpath('.//div[@class="location"]/text()')
print(product)
result = json.dumps(product, ensure_ascii=False) + ',\n'
with open('product.json', 'ab') as f:
f.write(result.encode('utf-8'))
# print(browser.get_cookies())
# print(browser.page_source)
wait = WebDriverWait(browser, 10)
MAX_PACE = 100
def main():
try:
browser.get('http://www.taobao.com')
print(browser.window_handles)
input = browser.find_element(By.ID, 'q')
# 搜索的信息
input.send_keys('iphone')
input.send_keys(Keys.ENTER)
browser.find_element(By.ID, 'J_Quick2Static').click()
time.sleep(1)
# 处理登录
weibo_login()
except:
main()
for i in range(1, MAX_PACE + 1):
index_page(i)
if __name__ == '__main__':
main()
总结:
1.学会利用cookie绕过验证码
2.学会从第三方进入需要爬取的网站
3.淘宝的滑动解锁(真的麻烦)
4.通过selenium模拟点击,爬取网站,虽然只爬了20页,但是思路没问题
淘宝必杀:滑动三连
小知识:
import json
a = {"name":"123","age":123}
text = json.dumps(a,ensure_ascii=False) + ",\n"
with open('1234.json','wb') as f:
f.write(text.encode('utf-8'))