forked from irisshakya/final_proj
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmethods.py
117 lines (93 loc) · 3.82 KB
/
methods.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
import pandas as pd
import csv
def us_comments():
'''
output:s
[
['XpVt6Z1Gjjo', "Logan Paul it's yo big day ‼️‼️‼️", '4', '0'],
['XpVt6Z1Gjjo', 'Say hi to Kong and maverick for me', '3', '0']
]
'''
with open('data/UScomments.csv', mode='r', encoding='utf-8-sig') as infile:
reader = csv.reader(infile, quoting=csv.QUOTE_ALL)
header_data = list(next(reader))
infile = infile.read().replace('\0', '').splitlines()
comment_data = list(csv.reader(infile))
#print(comment_data[963])
#return comment_data[963][1]
#print(comment_data[:5])
return comment_data
# to read count csv, should maintain usecols becasue commas are inside string, otherwise will crash
# i included to_csv to save the clipped data that works for us
def us_videos():
count_data = pd.read_csv('data/USvideos.csv', usecols=['video_id','title','views','likes','dislikes','comment_total'])
# return (count_data.iloc[963])
return count_data
def process_comment_data(comment_data):
'''
process_comment_data()
returns a list of unique video ids along with their comment count
'''
list_of_ids = []
comment_map = {}
for comment in comment_data:
# Append the video ID to the list_of_ids
list_of_ids.append(comment[0])
# Update the comment_map with the count of comments for each video ID
if comment[0] not in comment_map:
comment_map[comment[0]] = 1
else:
comment_map[comment[0]] += 1
# print(len(list_of_ids))
# print(len(list(set(list_of_ids))))
# print(len(comment_map))
# for key in comment_map:
# print(key, comment_map[key])
return list_of_ids, comment_map
def agregate_sentiments(comment_map, polarity_score_list):
'''
this function average score calculator for a video and a send it to a csv.
Args:
comment_map (hashMap(str, int)): has unique video_id and count of it
polarity_score_list(list of int): list of sentiment polarity score
Returns:
writes out a csv file=>
video_id,compound_score
zuKX0fPlo2Q,0.9394
UJKl7ToDi20,0.8954999999999999
dsH83p_mfEs,0.8493666666666666
'''
score_agregate = {}
#calulating sum of scores for a video
for comment in polarity_score_list:
vid_id = comment[0]
if vid_id not in score_agregate:
score_agregate[vid_id] = comment[2]
else:
score_agregate[vid_id] += comment[2]
# at this point the agregate value contains sums for each video
for vid_id in score_agregate:
score_agregate[vid_id] /= comment_map[vid_id]
# write the csv in descending order
top_items = sorted(score_agregate.items(), key=lambda x: x[1], reverse=True)
# write the items to a CSV file
with open('top_items.csv', mode='w', newline='') as file:
writer = csv.writer(file)
writer.writerow(['video_id', 'compound_score'])
for key, value in top_items:
writer.writerow([key, value])
def merger():
'''
simple merging function to get columns: video_id,views,likes,dislikes,comment_total,compound_score
'''
count_data = pd.read_csv('data/USvideos.csv', usecols=['video_id','views','likes','dislikes','comment_total'], index_col=None)
#print('count_data: ', type(count_data))
# Read the score CSV file
score_csv = pd.read_csv('top_items.csv', index_col=None)
#print('score_csv: ', type(score_csv))
# Merge the two dataframes on 'video_id' column
merged_df = pd.merge(count_data, score_csv, on='video_id')
# drop duplicates based on a specific column (e.g. 'column_name')
merged_df.drop_duplicates(subset=['video_id'], inplace=True)
# Save the merged dataframe as CSV
merged_df.to_csv('merged.csv', index=False)