实践 | 海量文件遍历，图像直方图统计，文本词频统计

机器学习-实践一：海量文件遍历

#!tree -L 3 ./data/

import zipfile
import os
def unzip_data(src_path,target_path):
    # 解压原始数据集，将src_path路径下的zip包解压至target_path目录下
    if(not os.path.isdir(target_path)):
        z = zipfile.ZipFile(src_path, 'r')
        z.extractall(path=target_path)
        z.close()

unzip_data('data/data10954/cat_12_test.zip','data/data10954/cat_12_test')
unzip_data('data/data10954/cat_12_train.zip','data/data55217/cat_12_train')

#!tree -L 6 ./data/

size_dict={}
type_dict={}

def get_size_type(path):
    files=os.listdir(path)
    for filename in files:
        temp_path=os.path.join(path,filename)
        if os.path.isdir(temp_path):#判断是不是文件夹
            get_size_type(temp_path)
        elif os.path.isfile(temp_path):
            type_name=os.path.splitext(temp_path)[1]#返回文件名称和后缀
            if not type_name:
                type_dict.setdefault("None",0)#不存在后缀名的时候加入一个None字典
                type_dict["None"]+=1
                size_dict.setdefault("None",0)
                size_dict["None"]+=os.path.getsize(temp_path)
            else:
                type_dict.setdefault(type_name,0);
                type_dict[type_name]+=1
                size_dict.setdefault(type_name,0)
                size_dict[type_name]+=os.path.getsize(temp_path)

path="data/"
get_size_type(path)
for each_type in type_dict.keys():
    print("%5s下共有[%5s]的文件[%5d]个，占用内存[%7.2f]MB"%(path,each_type,type_dict[each_type],size_dict[each_type]/(1024*1024)))
print("总文件数：[%d]"%(sum(type_dict.values())))
print("总内存大小：[%2f]GB"%(sum(size_dict.values()/(1024*3))))

机器学习-实践三：图像直方图统计

#总体图像直方图
import cv2
from matplotlib import pyplot as plt
img=cv2.imread('data/data131367/微信图片_20220308140220.jpg',1)
plt.hist(img.reshape([-1]),256,[0,256])#拉平图片
plt.show()

#三个通道图像直方图
import cv2
from matplotlib import pyplot as plt
img=cv2.imread('data/data131367/微信图片_20220308140220.jpg',1)#图片读到内存中
color=('b','g','r')#使用不同颜色
for i,col in enumerate(color):
    histr=cv2.calcHist([img],[i],None,[256],[0,256])
    plt.plot(histr,color=col)
    plt.xlim([0,256])

plt.show()
#红色偏右，图片整体红色强

机器学习-实践四：文本词频分析

#读取本文
with open('test.txt','r',encoding='UTF-8')as novelfile:
    novel=novelfile.read()

#文本分词
import jieba
novellist=list(jieba.lcut(novel))

#找到剔除停用词
stopwords=[line.strip() for line in open('stop.txt','r',encoding='UTF-8').readlines()]

noveldict={}
for word in novellist:
    if word not in stopwords:
        if len(word)==1:
            continue#长度为1的忽略
        else:
            noveldict[word]=noveldict.get(word,0)+1;

#根据词频排序
novellistsorted=list(noveldict.items())
novellistsorted.sort(key=lambda e:e[1],reverse=True)

#打印前二十的词
for topwordup in novellistsorted[:20]:
    print(topwordup)