对京东商品爬取的图片，基于机器学习进行分类

 fetchimg.py  #这个文件主要是对京东网站上的商品图片进行爬取

import requests
from bs4 import BeautifulSoup as bs
import time
x=0
pricelist=[]
categroy=[]
#爬取商品关键信息并存入Excel
def fashionDataScrape(m):
    global pricelist
    global categroy
    import pandas as pd
    n=0
    word=''
    searchStr=[]
    data3=pd.DataFrame()
    data1=pd.DataFrame()
    #逐一提取关键字
    a=eval(input("需要几个关键字："))
    for i in range(a):
        xm=input("请输入类型：")
        searchStr.append(xm)
    for ii in range(a):
        keyw=searchStr[ii]
        #开始批量爬取页面信息
        for i in range(1,m):
            url = 'https://search.jd.com/Search?keyword='+keyw+'&qrst=1&zx=1&pvid=11b9950a726e4367a1d21f58bd646ac3&page='+str(i)+'&s=61&click=0'
            headers={'User-Agent':"Dalvik/2.1.0 (Linux; U; Android 11; M2104K10AC Build/RP1A.200720.011)"}
            r=requests.get(url, headers=headers,verify=False,timeout = 30)
            r.encoding="utf-8"
            text=r.text
            import re
            soup=bs(text,'lxml')
            total=soup.find_all(name='div',attrs={'class':'gl-i-wrap'})
            for ii in total:
                price=ii.find(name='div',attrs={'class':'p-price'}).find(name='i').text
                img='https:'+re.findall('" data-lazy-img="(.*?)" height="220" width="220"/>',str(ii),re.S)[0]
                ida=keyw
                #获取商品图片信息
                fetchImgInfo(ida,img)
                desc=ii.find(name='div',attrs={'class':'p-name p-name-type-2'}).find(name='em').text+ii.find(name='div',attrs={'class':'p-shop'}).text.strip('\n')+ii.find(name='div',attrs={'class':'p-icons'}).text.strip('\n')
                #获取词云信息
                word+=desc
                import pandas as pd
                data=pd.DataFrame({'id':ida,
                                 'desc':desc,
                                 'price':price,
                                  'img':img
                                 },index=[n]
                                )
                data1=data1.append(data,ignore_index=True) 
                #列表储存价格和类别
                data2=pd.DataFrame({'类别':keyw,
                                    '价格':float(price)

                                 },index=[n]
                                )
                data3=data3.append(data2,ignore_index=True) 
                n+=1
    data1.to_excel("商品情况.xlsx", index=False)#这个储存位置需要改到自己的磁盘
    return searchStr
# 获取图片
def fetchImgInfo(ida,img):
    global x
    ida=ida
    urlimg=img
    header={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36'}
    r=requests.get(urlimg,headers=header,timeout=30)
    text=r.text
    img = requests.get(urlimg, headers=header).content
    path = "img\\"+str(x)+ ".jpg"
    # 将图片写入指定的目录 写入文件用"wb"
    with open(path, 'wb') as f:
        f.write(img)
        print("正在下载第{}张图片".format(x))
        x += 1
m=eval(input('要获取多少页信息（两页起步）：'))

 forecast.py  #对爬取来的图片进行分类及预测

import os
import cv2
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.model_selection import cross_val_score
from fetchimg import *
a=fashionDataScrape(m)
print(a)
# 第一步 切分训练集和测试集
X = []  # 定义图像名称
Y = []  # 定义图像分类类标
Z = []  # 定义图像像素
namelist=input
n=1
m=0
names = os.listdir('\img')
for i in names:
    #获取图片
    X.append('img\\' + i)
    # 获取图像类标即为文件夹名称
    #30个为一组分类
    if n%30!=0:
        b=a[m]
        Y.append(b)
        n+=1
    else:
        Y.append(b)
        n += 1
        m+=1

X = np.array(X)
Y = np.array(Y)
# 随机率为100% 选取其中的30%作为测试集
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=1)
print(len(X_train), len(X_test), len(y_train), len(y_test))
# 第二步 图像读取及转换为像素直方图
# 训练集
XX_train = []
for i in X_train:
    # 读取图像
    image = cv2.imread(i)

    # 图像像素大小一致
    img = cv2.resize(image, (256, 256), interpolation=cv2.INTER_CUBIC)

    # 计算图像直方图并存储至X数组
    hist = cv2.calcHist([img], [0, 1], None, [256, 256], [0.0, 255.0, 0.0, 255.0])

    XX_train.append(((hist / 255).flatten()))
# 测试集
XX_test = []
for i in X_test:
    # 读取图像
    # print i
    image = cv2.imread(i)
    # 图像像素大小一致
    img = cv2.resize(image, (256, 256), interpolation=cv2.INTER_CUBIC)
    # 计算图像直方图并存储至X数组
    hist = cv2.calcHist([img], [0, 1], None, [256, 256], [0.0, 255.0, 0.0, 255.0])
    XX_test.append(((hist / 255).flatten()))
# 第三步 基于KNN的图像分类处理
from sklearn.neighbors import KNeighborsClassifier  # 调用分类器

myknn = KNeighborsClassifier()
score = []
# for i in range(1,30):
clf = KNeighborsClassifier(n_neighbors=1).fit(XX_train, y_train)
predictions_labels = clf.predict(XX_test)
#     myknn.set_params(**{"n_neighbors":i})
#     score=cross_val_score(myknn,XX_test, y_test,scoring='accuracy',cv=5)
#     print(score)
print('算法评价:')
print((classification_report(y_test, predictions_labels)))

# 输出前10张图片及预测结果
k = 0
while k < 10:
    # 读取图像
    print(X_test[k])
    image = cv2.imread(X_test[k])
    print(predictions_labels[k])
    # 显示图像
    cv2.imshow("img", image)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
    k = k + 1