Tf–idf term weighting

$tf idf(t,d) = tf(t,d) * idf(t)$
where t means terms, d means documents.

1. Using the TfidfTransformer’s default settings ， smooth_idf=True

$idf(t) = log(\frac{n}{df(t)})+1$

TfidfTransformer(norm='l2', use_idf=True, smooth_idf=True, sublinear_tf=False)

2. Using the TfidfTransformer’s default settings ， smooth_idf=False

$idf(t) = log(\frac{n+1}{df(t)+1})+1$

TfidfTransformer(norm='l2', use_idf=True, smooth_idf=False, sublinear_tf=False)

注：最后对计算出的 $tf idf(t,d) = tf(t,d) * idf(t)$ 进行单位化 $v = \frac{v}{||v||}$

e.g.

# 如下三个词的词频矩阵
counts = [[3, 0, 1],
          [2, 0, 0],
          [3, 0, 0],
          [4, 0, 0],
          [3, 2, 0],
          [3, 0, 2]]
def compute_idf(ndarray):
    idfa = []
    n,m = ndarray.shape
    for i in range(m):
        df = list(ndarray[:,i]).count(0)
        # v = np.log(n/(n-df))+1 # smooth_idf=False
        v = np.log((1 + n) / (n-df + 1)) + 1 # smooth_idf=True
        idfa.append(v)
    return  idfa

def compute_tfidf(idfa,ndarray):
    n, m = ndarray.shape
    lst = np.array(np.zeros(shape=(n,m)))
    # print(lst,idfa,ndarray)
    for i in range(n):
        # lst[i, :] = idfa * ndarray[i, :]
        t = idfa * ndarray[i, :]
        lst[i,:] = t/np.sqrt(sum(t**2))
    print(lst)

idfa = compute_idf(np.asarray(counts))
print(idfa)
compute_tfidf(idfa,np.asarray(counts))

[[ 0.85151335  0.          0.52433293]
 [ 1.          0.          0.        ]
 [ 1.          0.          0.        ]
 [ 1.          0.          0.        ]
 [ 0.55422893  0.83236428  0.        ]
 [ 0.63035731  0.          0.77630514]]

等价于

counts = [[3, 0, 1],
          [2, 0, 0],
          [3, 0, 0],
          [4, 0, 0],
          [3, 2, 0],
          [3, 0, 2]]

from sklearn.feature_extraction.text import TfidfTransformer
transformer = TfidfTransformer()    # smooth_idf=True
tfidf = transformer.fit_transform(counts)
print(tfidf.toarray())

transformer = TfidfTransformer(smooth_idf=False)
tfidf = transformer.fit_transform(counts)
print(tfidf.toarray())