POS_IDF

#! /usr/bin/python

import os
import sys

from math import *
Delta = 0.1

npos_freq = {}
npos_file = open("./npos_freq.dat")
C = 0

for line in npos_file:
    line = line.strip()

    if not line:
        continue
    (npos,count) = line.split("\t")
    npos_freq[npos] = int(count)
    C += int(count)


for line in sys.stdin:
    line = line.strip()


    if not line:
        continue

    (term,info1) = line.split("\t",1)
    (digits,info2) = info1.split("|",1)

    (tf,pf) = digits.split(":",1)

    tf = float(tf)
    pf = float(pf)

    npos_info = info2.split(" ")

    pos_idf = log(float(C)/pf)
    pos_ridf = pos_idf + log(1+Delta-pow(2.71828,float(tf)/C))
    pos_bs = log(tf-pf + 1+Delta)
    
    boolean = 0
    weighted1 = 0
    weighted2 = 0
    for pair in npos_info:
        (npos,count) = pair.split(":")
        count = float(count)
        boolean += npos_freq[npos]/float(C)
        weighted1 += npos_freq[npos]/float(C) * 1.0/pf;
        weighted2 += npos_freq[npos]/float(C) * count/tf;


    print "%s|%f\t%f\t%f\t%f\t%f\t%f" %(term,pos_idf,pos_ridf,pos_bs,boolean,weighted1,weighted2)