python算法应用_FP-growth算法应用实例(基于python)

我把关于FP-Growth的算法原理,python实现代码,以及代码解读放在了另外一篇文章:有兴趣可以看看。

这篇文章给出该算法的一个很酷的实例应用。我们将用到一个叫 kosarak.dat 的数据集,可以从这里下载。这份数据集包含将近100万条记录,对于展示FP-Growth算法的速度十分有效。该文件的每一行包含某个用户浏览过的新闻报道。用户和报道被编码成整数。

为了看起来方便,还是先放一下python的实现代码:

#FP-Growth实现代码

class treeNode:

def __init__(self, nameValue, numOccur, parentNode):

self.name = nameValue

self.count = numOccur

self.nodeLink = None

self.parent = parentNode

self.children = {}

def inc(self, numOccur):

self.count += numOccur

def disp(self, ind=1):

print ' '*ind, self.name, ' ', self.count

for child in self.children.values():

child.disp(ind+1)

def updateHeader(nodeToTest, targetNode):

while nodeToTest.nodeLink != None:

nodeToTest = nodeToTest.nodeLink

nodeToTest.nodeLink = targetNode

def updateFPtree(items, inTree, headerTable, count):

if items[0] in inTree.children:

# 判断items的第一个结点是否已作为子结点

inTree.children[items[0]].inc(count)

else:

# 创建新的分支

inTree.children[items[0]] = treeNode(items[0], count, inTree)

# 更新相应频繁项集的链表,往后添加

if headerTable[items[0]][1] == None:

headerTable[items[0]][1] = inTree.children[items[0]]

else:

updateHeader(headerTable[items[0]][1], inTree.children[items[0]])

# 递归

if len(items) > 1:

updateFPtree(items[1::], inTree.children[items[0]], headerTable, count)

def createFPtree(dataSet, minSup=1):

headerTable = {}

for trans in dataSet:

for item in trans:

headerTable[item] = headerTable.get(item, 0) + dataSet[trans]

for k in headerTable.keys():

if headerTable[k] < minSup:

del(headerTable[k]) # 删除不满足最小支持度的元素

freqItemSet = set(headerTable.keys()) # 满足最小支持度的频繁项集

if len(freqItemSet) == 0:

return None, None

for k in headerTable:

headerTable[k] = [headerTable[k], None] # element: [count, node]

retTree = treeNode('Null Set', 1, None)

for tranSet, count in dataSet.items():

# dataSet:[element, count]

localD = {}

for item in tranSet:

if item in freqItemSet: # 过滤,只取该样本中满足最小支持度的频繁项

localD[item] = headerTable[item][0] # element : count

if len(localD) > 0:

# 根据全局频数从大到小对单样本排序

orderedItem = [v[0] for v in sorted(localD.items(), key=lambda p:p[1], reverse=True)]

# 用过滤且排序后的样本更新树

updateFPtree(orderedItem, retTree, headerTable, count)

return retTree, headerTable

def loadSimpDat():

simDat = [['r','z','h','j','p'],

['z','y','x','w','v','u','t','s'],

['z'],

['r','x','n','o','s'],

['y','r','x','z','q','t','p'],

['y','z','x','e','q','s','t','m']]

return simDat

# 构造成 element : count 的形式

def createInitSet(dataSet):

retDict={}

for trans in dataSet:

key = frozenset(trans)

if retDict.has_key(key):

retDict[frozenset(trans)] += 1

else:

retDict[frozenset(trans)] = 1

return retDict

# 数据集

def loadSimpDat():

simDat = [['r','z','h','j','p'],

['z','y','x','w','v','u','t','s'],

['z'],

['r','x','n','o','s'],

['y','r','x','z','q','t','p'],

['y','z','x','e','q','s','t','m']]

return simDat

# 构造成 element : count 的形式

def createInitSet(dataSet):

retDict={}

for trans in dataSet:

key = frozenset(trans)

if retDict.has_key(key):

retDict[frozenset(trans)] += 1

else:

retDict[frozenset(trans)] = 1

return retDict

# 递归回溯

def ascendFPtree(leafNode, prefixPath):

if leafNode.parent != None:

prefixPath.append(leafNode.name)

ascendFPtree(leafNode.parent, prefixPath)

# 条件模式基

def findPrefixPath(basePat, myHeaderTab):

treeNode = myHeaderTab[basePat][1] # basePat在FP树中的第一个结点

condPats = {}

while treeNode != None:

prefixPath = []

ascendFPtree(treeNode, prefixPath) # prefixPath是倒过来的,从treeNode开始到根

if len(prefixPath) > 1:

condPats[frozenset(prefixPath[1:])] = treeNode.count # 关联treeNode的计数

treeNode = treeNode.nodeLink # 下一个basePat结点

return condPats

def mineFPtree(inTree, headerTable, minSup, preFix, freqItemList):

# 最开始的频繁项集是headerTable中的各元素

bigL = [v[0] for v in sorted(headerTable.items(), key=lambda p:p[1])] # 根据频繁项的总频次排序

for basePat in bigL: # 对每个频繁项

newFreqSet = preFix.copy()

newFreqSet.add(basePat)

freqItemList.append(newFreqSet)

condPattBases = findPrefixPath(basePat, headerTable) # 当前频繁项集的条件模式基

myCondTree, myHead = createFPtree(condPattBases, minSup) # 构造当前频繁项的条件FP树

if myHead != None:

# print 'conditional tree for: ', newFreqSet

# myCondTree.disp(1)

mineFPtree(myCondTree, myHead, minSup, newFreqSet, freqItemList) # 递归挖掘条件FP树

运行:

#从新闻网站点击流中挖掘

parsedDat=[line.split() for line in open('kosarak.dat').readlines()]

initSet=createInitSet(parsedDat)

myFPtree,myHeaderTab=createFPtree(initSet,100000)

myFreqList=[]

mineFPtree(myFPtree,myHeaderTab,100000,set([]),myFreqList)

print "myFreqList\'s length: %s" % len(myFreqList)

for item in myFreqList:

print item

运行结果:

整个运行过程仅用了十几秒。

这个运行结果表明:有9个新闻报道或报道集合曾经被10万或者更多的人浏览过。这些报道或报道集合存储在变量myFreqList中。

  • 0
    点赞
  • 3
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值