python统计特定类型文件数量_Python 实现指定目录内重复文件统计

实现存储目录内重复文件以及重复文件个数的统计，输出统计文件

#!/usr/bin/python

# -*- coding: UTF-8 -*-

import os

import datetime

import platform

import sys

import  hashlib

def CalcFileSha256(filname):

''' calculate file sha256 '''

bufsize =1024 *1024 *16

with open(filname,"rb")as f:

sha256obj = hashlib.sha256()

while True:

data = f.read(bufsize)

if data  ==None or len(data) ==0:

break

sha256obj.update(data)

hash_value = sha256obj.hexdigest()

return hash_value

class CoSailTextFile():

def __init__(self, filename =None):

self.filename = filename

self.f =None

def open(self, m):

try:

self.f = open(self.filename, m)

except Exceptionas e:

print(e)

self.f =None

return False

return True

def writeLine(self, line):

try:

self.f.write(line +"\n")

return True

except Exceptionas  e:

print(e)

return False

def writeLines(self, lines):

for linein lines:

if self.writeLine(line) ==False:

return False

return True

def readLine(self):

text =None

try:

text = self.f.readline()

except Exceptionas e:

print(e)

return None

return text

def readAll(self):

return self.f.read()

def readLines(self):

return self.f.readlines()

def close(self):

if self.f !=None:

try:

self.f.close()

self.f =None

except Exceptionas e:

print(e)

return False

return True

return True

def isOpen(self):

return self.f !=None

def getFilePathExtend(fileName):

filePath, suffix = os.path.splitext(fileName)

filePath = os.path.dirname(fileName)

l = len(suffix)

fileName = os.path.split(fileName)[1]

return filePath,  suffix[1:l +1], fileName

filesdict = {}

def scansamenamefile(path):

if os.path.exists(path) ==False:

print("Store Path :" + path +" isn't exist!")

return None

try:

files = os.listdir(path)

except FileNotFoundError:

print("File Not FoundError")

return None

for fiin files:

# file full path

# sleep 1

# time.sleep(1)

fi_d = os.path.join(path, fi)

samefilelist =None

# is directory

if os.path.isdir(fi_d):

if os.path.islink(fi_d):

continue

""" recursion"""

scansamenamefile(fi_d)

# print("dir = ", fi_d)

# is file

else:

file_path, suffix, file_name = getFilePathExtend(fi_d)

tmpfullpath = file_path +"/" + file_name

try:

samefilelist = filesdict[file_name]

except Exceptionas e:

print(e)

samefilelist =None

if samefilelist ==None:

samefilelist = []

filesdict[file_name] = samefilelist

fileattrdict = {}

fileattrdict["filepath"] = tmpfullpath

fileattrdict["size"] = os.path.getsize(tmpfullpath)

fileattrdict["sha256"] = CalcFileSha256(tmpfullpath)

samefilelist.append(fileattrdict)

return filesdict

def statallfiles(d):

v =0;

o =None

for kin d:

o = d[k]

v += len(o)

return v

def statsamenamefiles(d):

v =0;

o =None

for kin d:

o = d[k]

if len(o) >=2:

v +=1

return v

def outduplicatefilemsg(d, textFile):

v =0;

o =None

textFile.writeLine("")

textFile.writeLine("**************Duplicate file list*****************")

for kin d:

o = d[k]

if len(o) >=2:

textFile.writeLine("")

txt ="file name : " + k +" Duplicate : " +str(len(o))

textFile.writeLine(txt)

txt ="file list :"

for objin o:

txt ="fileputh :" + obj["filepath"] +" size : " + str(obj["size"]) +" SHA256 = " + obj["sha256"]

textFile.writeLine(txt)

def outonefilemsg(d, textFile):

v =0;

o =None

textFile.writeLine("")

textFile.writeLine("**************Unique file list*****************")

for kin d:

o = d[k]

if len(o) ==1:

for objin o:

txt ="fileputh :" + obj["filepath"] +" size : " + str(obj["size"])

textFile.writeLine(txt)

def statpathfileattr(pathlistfilename, logpath):

"""

:param pathlistfilename:  要统计的文件目录列表名，每行一个目录    :param logpath 日志文件路径:return:

"""

"""

"""

if logpathis None:

"""

如果日志路径为空，则以输入文件列表路径为日志存储路径"""

logpath, _, _= getFilePathExtend(pathlistfilename)

if os.path.exists(pathlistfilename) ==False:

print("path list filename isn't exist!")

return

if os.path.exists(logpath) ==False:

print("log path isn't exist")

return

f = CoSailTextFile(pathlistfilename)

f.open("r")

lines = f.readLines()

for linein lines:

line = line.strip('\n')

statpathfileattrbypath(line, logpath)

def statpathfileattrbypath(path, logpath):

"""

统计指定路径的文件数量以及重复文件，并输出统计信息

:param path: 要统计的路径    :param logpath: 统计信息存储路基

:return:

"""

d = scansamenamefile(path)

if d ==None :

return

print("allfiles", statallfiles(d))

print("sameallfiles", statsamenamefiles(d))

t = path.replace("/","-");

t = t.replace("\\","-");

t = t.replace(":","")

if (platform.system() =='Windows'):

pass

elif (platform.system()=='Linux'):

t = t[1:]

lonfilename = logpath +"/" + t + datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') +".log"

lonfilename = lonfilename.replace("//","/")

f = CoSailTextFile(lonfilename)

f.open("w")

f.writeLine("stat path : " + path)

f.writeLine("all files : " + str(statallfiles(d)))

f.writeLine("Duplicate files : " + str(statsamenamefiles(d)))

outduplicatefilemsg(d, f)

outonefilemsg(d, f)

f.close()

#print("file_name : ", file_path + "/" + file_name)

if __name__ =='__main__':

filesdict.clear()

filesdict={}

argv = sys.argv

print(argv)

if (len(argv) ==3):

statpathfileattr(argv[1], argv[2])

elif (len(argv) ==2):

statpathfileattr(argv[1],None)

else:

statpathfileattr("E:/1.txt","E:/")

print(type(CalcFileSha256("E:/gsl2.4.zip")))

sha2561 =CalcFileSha256("E:/gsl2.4.zip")

sha2562 = CalcFileSha256("E:/1.txt")

if (sha2561 == sha2562):

print("true")

print(sha2562)

print(sha2561)

#d = buildfilenamedictformfilelist("/home/hadoop/tmp/dir-and-files.list")

"""存储目的路径根目录"""

"""

destrootpath = "/home/hadoop/tmp4"

path = "E:/data"

d =  scansamenamefile(path)

print(d)

for k in d:

print("key =", k)

print("files = ", len(d[k]))

print("allfiles", statallfiles(d))

print("sameallfiles", statsamenamefiles(d))

f = CoSailTextFile("e:/sss.log")

f.open("w")

f.writeLine("stat path : " + path)

f.writeLine("all files : " + str(statallfiles(d)))

f.writeLine("Duplicate files : " +str(statsamenamefiles(d)))

outduplicatefilemsg(d, f)

outonefilemsg(d, f)

f.close()

f = CoSailTextFile("E:/1.txt")

f.open("r")

text = f.readLines()

print((text))

"""