参考爬虫初学6
本来就想爬一下京东和淘宝,选择有点保证的店铺来买小叶紫檀(水太深),现在初步通过爬京东找了下,也发现一些可信度比较高的店铺,奈何价格不太合我心意,我还是找我的几十块钱的手串吧
import requests
from lxml import etree
import time
import csv
# 定义函数抓取每页前30条商品信息
def crow_first(n):
# 构造每一页的url变化
url = 'https://search.jd.com/Search?keyword=%E5%B0%8F%E5%8F%B6%E7%B4%AB%E6%AA%80&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E5%B0%8F%E5%8F%B6%E7%B4%AB%E6%AA%80&psort=3&cid2=12041&cid3=12047&ev=7372_81416%5E&page=' + str(
2 * n - 1)
head = {'authority': 'search.jd.com',
'method': 'GET',
'path': '/s_new.php?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA&cid2=653&cid3=655&page=4&s=84&scrolling=y&log_id=1529828108.22071&tpl=3_M&show_items=7651927,7367120,7056868,7419252,6001239,5934182,4554969,3893501,7421462,6577495,26480543553,7345757,4483120,6176077,6932795,7336429,5963066,5283387,25722468892,7425622,4768461',
'scheme': 'https',
'referer': 'https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA&cid2=653&cid3=655&page=3&s=58&click=0',
'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36',
'x-requested-with': 'XMLHttpRequest',
'Cookie': ''
}
r = requests.get(url, headers=head)
# 指定编码方式,不然会出现乱码