fetchimg.py #这个文件主要是对京东网站上的商品图片进行爬取
import requests
from bs4 import BeautifulSoup as bs
import time
x=0
pricelist=[]
categroy=[]
#爬取商品关键信息并存入Excel
def fashionDataScrape(m):
global pricelist
global categroy
import pandas as pd
n=0
word=''
searchStr=[]
data3=pd.DataFrame()
data1=pd.DataFrame()
#逐一提取关键字
a=eval(input("需要几个关键字:"))
for i in range(a):
xm=input("请输入类型:")
searchStr.append(xm)
for ii in range(a):
keyw=searchStr[ii]
#开始批量爬取页面信息
for i in range(1,m):
url = 'https://search.jd.com/Search?keyword='+keyw+'&qrst=1&zx=1&pvid=11b9950a726e4367a1d21f58bd646ac3&page='+str(i)+'&s=61&click=0'
headers={'User-Agent':"Dalvik/2.1.0 (Linux; U; Android 11; M2104K10AC Build/RP1A.200720.011)"}
r=requests.get(url, headers=headers,verify=False,timeout = 30)
r.encoding="utf-8"
text=r.text
import re
soup=bs(text,'lxml')
total=soup.find_all(name='div',attrs={'class':'gl-i-wrap'})
for ii in total:
price=ii.find(name='div',attrs={'class':'p-price'}).find(name='i').text
img='https:'+re.findall('" data-lazy-img="(.*?)" height="220" width="220"/>',str(ii),re.S)[0]
ida=keyw
#获取商品图片信息
fetchImgInfo(ida,img)
desc=ii.find(name='div',attrs={'class':'p-name p-name-type-2'}).find(name='em').text+ii.find(name='div',attrs={'class':'p-shop'}).text.strip('\n')+ii.find(name='div',attrs={'class':'p-icons'}).text.strip('\n')
#获取词云信息
word+=desc
import pandas as pd
data=pd.DataFrame({'id':ida,
'desc':desc,
'price':price,
'img':img
},index=[n]
)
data1=data1.append(data,ignore_index=True)
#列表储存价格和类别
data2=pd.DataFrame({'类别':keyw,
'价格':float(price)
},index=[n]
)
data3=data3.append(data2,ignore_index=True)
n+=1
data1.to_excel("商品情况.xlsx", index=False)#这个储存位置需要改到自己的磁盘
return searchStr
# 获取图片
def fetchImgInfo(ida,img):
global x
ida=ida
urlimg=img
header={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36'}
r=requests.get(urlimg,headers=header,timeout=30)
text=r.text
img = requests.get(urlimg, headers=header).content
path = "img\\"+str(x)+ ".jpg"
# 将图片写入指定的目录 写入文件用"wb"
with open(path, 'wb') as f:
f.write(img)
print("正在下载第{}张图片".format(x))
x += 1
m=eval(input('要获取多少页信息(两页起步):'))
forecast.py #对爬取来的图片进行分类及预测
import os
import cv2
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.model_selection import cross_val_score
from fetchimg import *
a=fashionDataScrape(m)
print(a)
# 第一步 切分训练集和测试集
X = [] # 定义图像名称
Y = [] # 定义图像分类类标
Z = [] # 定义图像像素
namelist=input
n=1
m=0
names = os.listdir('\img')
for i in names:
#获取图片
X.append('img\\' + i)
# 获取图像类标即为文件夹名称
#30个为一组分类
if n%30!=0:
b=a[m]
Y.append(b)
n+=1
else:
Y.append(b)
n += 1
m+=1
X = np.array(X)
Y = np.array(Y)
# 随机率为100% 选取其中的30%作为测试集
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=1)
print(len(X_train), len(X_test), len(y_train), len(y_test))
# 第二步 图像读取及转换为像素直方图
# 训练集
XX_train = []
for i in X_train:
# 读取图像
image = cv2.imread(i)
# 图像像素大小一致
img = cv2.resize(image, (256, 256), interpolation=cv2.INTER_CUBIC)
# 计算图像直方图并存储至X数组
hist = cv2.calcHist([img], [0, 1], None, [256, 256], [0.0, 255.0, 0.0, 255.0])
XX_train.append(((hist / 255).flatten()))
# 测试集
XX_test = []
for i in X_test:
# 读取图像
# print i
image = cv2.imread(i)
# 图像像素大小一致
img = cv2.resize(image, (256, 256), interpolation=cv2.INTER_CUBIC)
# 计算图像直方图并存储至X数组
hist = cv2.calcHist([img], [0, 1], None, [256, 256], [0.0, 255.0, 0.0, 255.0])
XX_test.append(((hist / 255).flatten()))
# 第三步 基于KNN的图像分类处理
from sklearn.neighbors import KNeighborsClassifier # 调用分类器
myknn = KNeighborsClassifier()
score = []
# for i in range(1,30):
clf = KNeighborsClassifier(n_neighbors=1).fit(XX_train, y_train)
predictions_labels = clf.predict(XX_test)
# myknn.set_params(**{"n_neighbors":i})
# score=cross_val_score(myknn,XX_test, y_test,scoring='accuracy',cv=5)
# print(score)
print('算法评价:')
print((classification_report(y_test, predictions_labels)))
# 输出前10张图片及预测结果
k = 0
while k < 10:
# 读取图像
print(X_test[k])
image = cv2.imread(X_test[k])
print(predictions_labels[k])
# 显示图像
cv2.imshow("img", image)
cv2.waitKey(0)
cv2.destroyAllWindows()
k = k + 1