python iphone 爬虫_初学爬虫之爬取某电商IPHONE信息

#!/usr/bin/env python#-*- coding: utf-8 -*-#@Time : 2019/12/30 10:51#@Site :#@File : jd_iphone.py#@Software: PyCharm

importosimportjsonimporttimeimporturllib3importloggingimportrequestsimportxlwt#import xlsxwriter as xlwt

from pyquery importPyQueryfrom selenium importwebdriver

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)#fidder抓包忽略请求warning

headers ={"Referer": "https://search.jd.com/","User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",

}#控制台日志输出

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

logger_page= logging.getLogger("jd_iphone_page")

logger_detail= logging.getLogger("jd_iphone_detail")defget_page_detail(maxp):

product_list=[]

p_no=0for page in range(1,2*maxp,2):

url= 'https://search.jd.com/Search?keyword=iphoneapple&page='+str(page)+'&click=0'#筛选iPhone手机链接

#resp = requests.get(url,headers=headers,verify=False)

#浏览器无窗模式

chrome_options =webdriver.ChromeOptions()

chrome_options.add_argument('--headless')

driver= webdriver.Chrome(chrome_options=chrome_options)#driver = webdriver.Chrome()

driver.get(url)

driver.implicitly_wait(10)#执