学习实例淘宝页面爬取
#程序的结构设计
步骤1:提交商品搜索请求,循环获取页面
步骤2:对于每个页面,提取商品名称和价格信息
步骤3:将信息国人出到屏幕上
步骤4:将数据存入数据库
#实例
import requests
import re
from sqlalchemy import create_engine
def gethtmltext(url):
try:
r=requests.get(url,timeout=30)
r.raise_for_status()
r.encoding=r.apparent_encoding
return r.text
except:
return ''
def parsepage(ilt,html):
try:
plt=re.findall(r'\"view_price\"\:\"[\d\.]*\"',html)
tlt=re.re.findall(r'\"raw_title\"\:\".*?\"',html)
for i in range(len(plt)):
price=eval(plt[i].split(':')[1])
title=eval(tit[i].split(':')[1])
ilt.append([price,title])
except:
print('')
def printgoodslist(ilt):
tplt="{:4}\t{:8}\t{:16}"
print(tplt.format('NO','price','name'))
count=0
for g in ilt:
count=count+1
print(tplt.format(count,g[0],g[1]))
print('')
def main():
goods='书包'
depth=2
start_url='https://s.taobao.com/search?q='+goods
infolist=[]
for i in range(depth):
try:
print('***geturl***')
url=start_url+'&s='+str(i*44)
print(url)
print('***gethtml***')
html=gethtmltext(url)
print(len(html))
print(html)
print('***findhtml***')
parsepage(infolist,html)
printgoodslist(infolist)
except:
continue
main()