-
Notifications
You must be signed in to change notification settings - Fork 3
/
tf_idf_calculate_new.py
110 lines (87 loc) · 3.3 KB
/
tf_idf_calculate_new.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# Author: Yuanwei Wu & Yang
# Date: 3/16/2016, recove 3/22/2016, revised 4/3/2016, update normalize: 4/23/2016
# Description: input: tf, df in list,
# output: calculate idf= log10(N/df), only output tf*idf in list
import math
import utils
#by Yang Tian
def tfidf_weight_calculation(tf_dic,df_dic,boolean_dic,doc_num):
tfidf_dic = {}
for key,value in tf_dic.iteritems():
#print key
tfidf_dic[key] = [ x*math.log10(float(doc_num)/df_dic[key]) for x in value]
tfidf = sorted([(k,v) for k,v in tfidf_dic.items()])
# # change the tfidt into list, remove the keys
tfidf_lst = list()
for k3,v3 in tfidf:
tfidf_lst.append(v3)
## save the tfidf: remove the key, and each list in represents a document vector
tfidf_doc = list(map(list, zip(*tfidf_lst)))
utils.store_datastructure('tfidf_matrix.pkl',tfidf_dic)
"""df_list = sorted(df_dic.items())
tfidf_list = sorted(tfidf_dic.items())
boolean_list = sorted(boolean_dic.items())
term_info = []
df_info = []
tf_info = []
bool_info = []
for x in df_list:
term_info.append(x[0])
df_info.append(x[1])
for y in tfidf_list:
tf_info.append(y[1])
for z in boolean_list:
bool_info.append(z[1])
tuple_info = zip(term_info,df_info,bool_info,tf_info)
#utils.writedic_to_csv(tuple_info,'tfdf_info.csv')"""
def tf_idf_calculation(tf, df):
tfidf = dict()
idf = dict()
tffile_length = len(tf[0][1])# get the length of documents
for key, value in df:
newidf = math.log10(float(tffile_length)/value)
idf.update({key:newidf}) # update idf dict
idf = [(v,k) for v,k in idf.items()] # change idf(dict type) to list of tuples
# calculat the idf*tf value
for k1,v1 in tf:
for k2,v2 in idf:
if k1==k2:
newtfidf = [v2*num for num in v1]
tfidf.update({k2:newtfidf}) # update tfidf dict
tfidf = sorted([(k,v) for k,v in tfidf.items()])
# # change the tfidt into list, remove the keys
tfidf_lst = list()
for k3,v3 in tfidf:
tfidf_lst.append(v3)
## save the tfidf: remove the key, and each list in represents a document vector
tfidf_doc = list(map(list, zip(*tfidf_lst)))
return tfidf_doc
"""
word_dic = utils.read_datastructure('word_dictonary.pkl')
df_dic = utils.read_datastructure('df_dictonary.pkl')
boolean_dic = utils.read_datastructure('boolean_dictonary.pkl')
tfidf_weight_calculation(word_dic,df_dic,boolean_dic,len(word_dic))
print('Done.')
"""
#print(len(word_dic['addit']))
#utils.writedic_to_csv(tf_list,'tf_test.csv')
#utils.readcsv('test.csv')
#correct
"""
df_file = utils.read_datastructure('df_dict.pkl')
tf_file = utils.read_datastructure('word_dict.pkl')
doc_tfidf_matrix = tf_idf_calculation(tf_file, df_file)
utils.store_datastructure('doc_tfidf_matrix.pkl',doc_tfidf_matrix)
"""
# doc_tfidf_distance = [math.sqrt(sum(val)) for val in doc_tfidf_matrix]
# utils.store_datastructure('doc_tfidf_distance.pkl',doc_tfidf_distance)
# doc_tfidf_normalize = []
# filelen = len(doc_tfidf_matrix)
# for i in xrange(0,filelen):
# v2 = doc_tfidf_distance[i]
# v1 = doc_tfidf_matrix[i]
# doc_tfidf_normalize.append([v1/v2 for v1 in doc_tfidf_matrix[i]])
# utils.store_datastructure('doc_tfidf_normalize.pkl',doc_tfidf_normalize)
# print(len(docTfidfMatrix[0])) # each list in docTfidfMatrix is 1*16287,
# 16287 is the number of terms
# print(docTfidfMatrix)