【笔记】手敲版 TF IDF

注1：

1.  涉及计算向向量夹角  【笔记】向量点乘（内积）和叉乘（外积、向量积）：对两个向量执行点乘运算，是对这两个向量对应位一一相乘之后求和的操作，点乘的结果是一个标量；叉乘结果是一个向量，它垂直于两向量构成的平面（法向量）_程序猿的探索之路的博客-CSDN博客_两个矢量的点乘注：行列式知识点：正文：https://blog.csdn.net/nyist_yangguang/article/details/121801944?ops_request_misc=%257B%2522request%255Fid%2522%253A%2522165258565416782395362014%2522%252C%2522scm%2522%253A%252220140713.130102334.pc%255Fblog.%2522%257D&request_id=165258565416782395362014&biz_id=0&utm_medium=distribute.pc_search_result.none-task-blog-2~blog~first_rank_ecpm_v1~rank_v31_ecpm-1-121801944-null-null.nonecase&utm_term=%E5%86%85%E7%A7%AF&spm=1018.2226.3001.4450

2. set() 会将数据元素打乱

注2：

         tf * idf 得到各自文章的句向量，我们现在需要用新的句子（句向量）来计算与原来各个句向量的夹角大小。我们在计算新句子的 tf 的基础上，来扩展原来句子的 idf 向量长度，使得它们可以进行矩阵乘法。

import numpy as np
from collections import Counter
import itertools
from visual import show_tfidf   # this refers to visual.py in my [repo](https://github.com/MorvanZhou/NLP-Tutorials/)

docs = [
    "it it is a good day, I like to stay here",
    "I am happy to be here",
    "I am bob",
    # "it is sunny today",
    # "I have a party today",
    # "it is a dog and that is a cat",
    # "there are dog and cat on the tree",
    # "I study hard this morning",
    # "today is a good day",
    # "tomorrow will be a good day",
    # "I like coffee, I like book and I like apple",
    # "I do not like it",
    # "I am kitty, I like bob",
    # "I do not care who like bob, but I like kitty",
    # "It is coffee time, bring your cup",
]

docs_words = [d.replace(",","").split(" ") for d in docs]

vocab = sorted(set(itertools.chain(*docs_words)))

v2i = {v: i for i, v in enumerate(vocab)}
i2v = {i: v for v, i in v2i.items()}


def safe_log(x):
    mask = x != 0
    x[mask] = np.log(x[mask])
    return x


tf_methods = {
        "log": lambda x: np.log(1+x),
        "augmented": lambda x: 0.5 + 0.5 * x / np.max(x, axis=1, keepdims=True),
        "boolean": lambda x: np.minimum(x, 1),
        "log_avg": lambda x: (1 + safe_log(x)) / (1 + safe_log(np.mean(x, axis=1, keepdims=True))),
    }
idf_methods = {
        "log": lambda x: 1 + np.log(len(docs) / (x+1)),
        "prob": lambda x: np.maximum(0, np.log((len(docs) - x) / (x+1))),
        "len_norm": lambda x: x / (np.sum(np.square(x))+1),
    }


def get_tf(method="log"):
    # term frequency: how frequent a word appears in a doc
    _tf = np.zeros((len(vocab), len(docs)), dtype=np.float64)    # [n_vocab, n_doc]
    for i, d in enumerate(docs_words):
        counter = Counter(d)
        for v in counter.keys():
            _tf[v2i[v], i] = counter[v] / counter.most_common(1)[0][1]
    weighted_tf = tf_methods.get(method, None)
    if weighted_tf is None:
        raise ValueError
    return weighted_tf(_tf)


def get_idf(method="log"):
    # inverse document frequency: low idf for a word appears in more docs, mean less important
    df = np.zeros((len(i2v), 1))
    for i in range(len(i2v)):
        d_count = 0
        for d in docs_words:
            d_count += 1 if i2v[i] in d else 0
        df[i, 0] = d_count

    idf_fn = idf_methods.get(method, None)
    if idf_fn is None:
        raise ValueError
    return idf_fn(df)


def cosine_similarity(q, _tf_idf):
    print(q.shape,q)
    print(_tf_idf.shape,_tf_idf)
    input()
    # unit_q = q / np.sqrt(np.sum(np.square(q), axis=0, keepdims=True))
    # unit_ds = _tf_idf / np.sqrt(np.sum(np.square(_tf_idf), axis=0, keepdims=True))
    print(_tf_idf.T.dot(q))
    # print(unit_ds.T.dot(unit_q))
    similarity=_tf_idf.T.dot(q).ravel()
    # similarity = unit_ds.T.dot(unit_q).ravel()
    print(similarity)
    input()
    return similarity


def docs_score(q, len_norm=False):
    q_words = q.replace(",", "").split(" ")

    # add unknown words
    unknown_v = 0
    for v in set(q_words):
        if v not in v2i:
            v2i[v] = len(v2i)
            i2v[len(v2i)-1] = v
            unknown_v += 1
    if unknown_v > 0:
        _idf = np.concatenate((idf, np.zeros((unknown_v, 1), dtype=np.float)), axis=0)
        _tf_idf = np.concatenate((tf_idf, np.zeros((unknown_v, tf_idf.shape[1]), dtype=np.float)), axis=0)
    else:
        _idf, _tf_idf = idf, tf_idf
    counter = Counter(q_words)
    q_tf = np.zeros((len(_idf), 1), dtype=np.float)     # [n_vocab, 1]
    for v in counter.keys():
        q_tf[v2i[v], 0] = counter[v]

    q_vec = q_tf * _idf            # [n_vocab, 1]

    q_scores = cosine_similarity(q_vec, _tf_idf)
    if len_norm:
        len_docs = [len(d) for d in docs_words]
        q_scores = q_scores / np.array(len_docs)
    return q_scores


def get_keywords(n=2):
    for c in range(3):
        col = tf_idf[:, c]
        idx = np.argsort(col)[-n:]
        print("doc{}, top{} keywords {}".format(c, n, [i2v[i] for i in idx]))


tf = get_tf()           # [n_vocab, n_doc]
idf = get_idf()         # [n_vocab, 1]
tf_idf = tf * idf       # [n_vocab, n_doc]
print("tf shape(vecb in each docs): ", tf.shape)
print("\ntf samples:\n", tf[:2])
print("\nidf shape(vecb in all docs): ", idf.shape)
print("\nidf samples:\n", idf[:2])
print("\ntf_idf shape: ", tf_idf.shape)
print("\ntf_idf sample:\n", tf_idf[:2])


# test
get_keywords()
q = "I get a coffee cup"
scores = docs_score(q)
d_ids = scores.argsort()[-3:][::-1]
print("\ntop 3 docs for '{}':\n{}".format(q, [docs[i] for i in d_ids]))

show_tfidf(tf_idf.T, [i2v[i] for i in range(tf_idf.shape[0])], "tfidf_matrix")

tf shape(vecb in each docs):  (14, 3)

tf samples:
 [[0.40546511 0.69314718 0.69314718]
 [0.40546511 0.         0.        ]]

idf shape(vecb in all docs):  (14, 1)

idf samples:
 [[0.71231793]
 [1.40546511]]

tf_idf shape:  (14, 3)

tf_idf sample:
 [[0.28882007 0.49374116 0.49374116]
 [0.56986706 0.         0.        ]]
doc0, top2 keywords ['stay', 'it']
doc1, top2 keywords ['be', 'happy']
doc2, top2 keywords ['am', 'bob']
(17, 1) [[0.71231793]
 [1.40546511]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]
 [0.        ]]
(17, 3) [[0.28882007 0.49374116 0.49374116]
 [0.56986706 0.         0.        ]
 [0.         0.69314718 0.69314718]
 [0.         0.97419418 0.        ]
 [0.         0.         0.97419418]
 [0.56986706 0.         0.        ]
 [0.56986706 0.         0.        ]
 [0.         0.97419418 0.        ]
 [0.40546511 0.69314718 0.        ]
 [0.56986706 0.         0.        ]
 [0.97419418 0.         0.        ]
 [0.56986706 0.         0.        ]
 [0.56986706 0.         0.        ]
 [0.40546511 0.69314718 0.        ]
 [0.         0.         0.        ]
 [0.         0.         0.        ]
 [0.         0.         0.        ]]


[[1.00665998]
 [0.35170068]
 [0.35170068]]
[1.00665998 0.35170068 0.35170068]

top 3 docs for 'I get a coffee cup':
['it it is a good day, I like to stay here', 'I am bob', 'I am happy to be here']

def show_tfidf(tfidf, vocab, filename):
    # [n_doc, n_vocab]
    plt.imshow(tfidf, cmap="YlGn", vmin=tfidf.min(), vmax=tfidf.max())
    plt.xticks(np.arange(tfidf.shape[1]), vocab, fontsize=6, rotation=90)
    plt.yticks(np.arange(tfidf.shape[0]), np.arange(1, tfidf.shape[0]+1), fontsize=6)
    plt.tight_layout()
    # creating the output folder 
    output_folder = './visual/results/'
    os.makedirs(output_folder, exist_ok=True)
    plt.savefig(os.path.join(output_folder, '%s.png') % filename, format="png", dpi=500)
    plt.show()