-
Notifications
You must be signed in to change notification settings - Fork 0
/
broken_link_checker_final.py
277 lines (184 loc) · 9.51 KB
/
broken_link_checker_final.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
# # Libraries
from usp.tree import sitemap_tree_for_homepage
from bs4 import BeautifulSoup
import requests
import pandas as pd
import json
from datetime import datetime
import os
# # Lists & Other Necessities
# Domain
fullDomain = 'https://tilburgsciencehub.com'
#Sitemap listpages
listPages_Raw = []
listPages = []
#Links on pages
externalLinksListRaw = []
uniqueExternalLinks = []
#broken link list
brokenLinksList = []
brokenLinksDict = {'link':[],'statusCode':[]}
#broken link location list
brokenLinkLocation = []
#set user agent
user_agent = {'User-agent': 'Mozilla/5.0'}
#git token by git secret
token = os.environ['GIT_TOKEN']
#git headers authorization
headers = {"Authorization" : "token {}".format(token)}
#Generate target repositoryURL using Github API
username = 'tilburgsciencehub'
Repositoryname = 'broken-link-checker'
url = "https://api.github.com/repos/{}/{}/issues".format(username,Repositoryname)
#github table setup
tablehead = "| URL | Broken Link | Anchor | Code |" + "\n" + "| ------------- | ------------- | ------------- | ------------- |" + "\n"
# # Functions
#Put all pages from sitemap in listPages_Raw
def getPagesFromSitemap(fullDomain):
listPages_Raw.clear()
tree = sitemap_tree_for_homepage(fullDomain)
for page in tree.all_pages():
listPages_Raw.append(page.url)
# Go through List Pages Raw output a list of unique pages links
def getListUniquePages():
listPages.clear()
for page in listPages_Raw:
if page in listPages:
pass
else:
listPages.append(page)
#get all links per page and insert url, dest. url & anchor text in externalLinksListRaw
def ExternalLinkList(listPages, fullDomain):
externalLinksListRaw.clear()
for url in listPages:
request = requests.get(url, headers=user_agent)
content = request.content
soup = BeautifulSoup(content, 'html.parser')
list_of_links = soup.find_all("a")
for link in list_of_links:
try:
if link["href"].startswith("#") or link["href"].startswith("mail"):
pass
else:
if link["href"] == '':
externalLinksListRaw.append([url,'Same destination as page',link.text])
elif link["href"].startswith("/"):
linkhrefadjust = link["href"][1:]
domain = fullDomain
urlcomp = domain + linkhrefadjust
externalLinksListRaw.append([url,urlcomp,link.text])
elif link["href"].startswith("../"):
if link["href"].startswith("../../"):
if link["href"].startswith("../../../"):
subsetlinkhref = link["href"].replace("../", '')
lastpartdomain = url.rsplit('/')[-4] + '/' + url.rsplit('/')[-3] + '/' + url.rsplit('/')[-2] + '/'
subseturl = url.replace(lastpartdomain, '')
newlink = subseturl + subsetlinkhref
externalLinksListRaw.append([url,newlink,link.text])
else:
subsetlinkhref = link["href"].replace("../", '')
lastpartdomain = url.rsplit('/')[-3] + '/' + url.rsplit('/')[-2] + '/'
subseturl = url.replace(lastpartdomain, '')
newlink = subseturl + subsetlinkhref
externalLinksListRaw.append([url,newlink,link.text])
else:
subsetlinkhref = '/' + link["href"].replace("../", '')
lastpartdomain = '/' + url.rsplit('/')[-2] + '/'
subseturl = url.replace(lastpartdomain, '')
newlink = subseturl + subsetlinkhref
externalLinksListRaw.append([url,newlink,link.text])
elif link["href"].startswith("./../"):
subsetlinkhref = link["href"].replace("./../", '')
lastpartdomain = url.rsplit('/')[-2] + '/'
subseturl = url.replace(lastpartdomain, '')
newlink = subseturl + subsetlinkhref
externalLinksListRaw.append([url,newlink,link.text])
elif '.py' in link["href"] and 'http://' not in link["href"]:
pass
else:
externalLinksListRaw.append([url,link["href"],link.text])
except:
pass
# Go through externalLinksListRaw output and create a list(uniqueExternalLinks) of unique pages links
def getUniqueExternalLinks(externalLinksListRaw):
uniqueExternalLinks.clear()
for link in externalLinksListRaw:
if link[1] in uniqueExternalLinks:
pass
else:
uniqueExternalLinks.append(link[1])
#identify Broken Links
def identifyBrokenLinks(uniqueExternalLinks):
brokenLinksList.clear()
count = 0
length_uniqueExternalLinks = len(uniqueExternalLinks)
for link in uniqueExternalLinks:
count = count + 1
print("Checking external link #",count," out of ",length_uniqueExternalLinks,".")
try:
statusCode = requests.get(link, headers=user_agent).status_code
if statusCode == 404:
brokenLinksDict['link'].append(link)
brokenLinksDict['statusCode'].append(statusCode)
brokenLinksList.append(link)
elif statusCode != 404 and statusCode > 399 and statusCode < 452:
brokenLinksDict['link'].append(link)
brokenLinksDict['statusCode'].append(statusCode)
brokenLinksList.append(link)
else:
pass
except:
brokenLinksDict['link'].append(link)
brokenLinksDict['statusCode'].append(statusCode)
brokenLinksList.append(link)
# Identify Unique Broken Links and Matches them to Original List of All External Links
def matchBrokenLinks(brokenLinksList,externalLinksListRaw):
global EndDataFrame
brokenLinkLocation.clear()
for link in externalLinksListRaw:
if link[1] in brokenLinksList:
brokenLinkLocation.append([link[0],link[1],link[2]])
else:
pass
dataframeFinal = pd.DataFrame(brokenLinkLocation,columns=["URL","Broken_Link_URL","Anchor Text"])
dataframeFinal2 = pd.DataFrame(brokenLinksDict)
EndDataFrame = dataframeFinal.merge(dataframeFinal2, left_on='Broken_Link_URL', right_on ='link', how='outer')
del EndDataFrame['link']
def push_issue_git():
#set dt_string with current date/time
now = datetime.now()
dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
#github issue title
titleissue = 'Broken/Error Links on ' + dt_string
#reset EndDataFrame index to itterate through
df3 = EndDataFrame.reset_index() # make sure indexes pair with number of rows
#empty table for git issue
table = ''
if len(df3.index) > 0:
#for each row in df3, add new row in git table
for index, row in df3.iterrows():
if '\n' in row['Anchor Text']:
anchortext = row['Anchor Text'].replace("\n", '')
tablebody = '|' + row['URL'] + '|' + row['Broken_Link_URL'] + '|' + anchortext + '|' + str(row['statusCode']) + '|' + '\n'
table = table + tablebody
else:
tablebody = '|' + row['URL'] + '|' + row['Broken_Link_URL'] + '|' + row['Anchor Text'] + '|' + str(row['statusCode']) + '|' + '\n'
table = table + tablebody
#create content of issue
tablecomp = tablehead + table
issuebody = 'Today, a total of ' + str(len(df3.index)) + ' link errors have been found. The following links have been found containing errors:' + '\n' + tablecomp
#defining data to push to git issue
data = {"title": titleissue, "body": issuebody, "assignee": "thierrylahaije"}
#Post issue message using requests and json
requests.post(url,data=json.dumps(data),headers=headers)
print('Process succeeded')
else:
pass
# # Execute Functions
getPagesFromSitemap(fullDomain)
getListUniquePages()
ExternalLinkList(listPages, fullDomain)
getUniqueExternalLinks(externalLinksListRaw)
identifyBrokenLinks(uniqueExternalLinks)
matchBrokenLinks(brokenLinksList,externalLinksListRaw)
push_issue_git()