Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sourcery Starbot ⭐ refactored DucarrougeR/themis-1 #1

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Sourcery Starbot ⭐ refactored DucarrougeR/themis-1 #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 23 additions & 23 deletions modules/connectivity.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,22 @@ def test_db_connections(self):
db_errors = []

for connection in all_connections:
if connection.name not in ('looker', 'looker__ilooker', 'looker__internal__analytics'):
if connection.dialect:
connection_tests = {'tests': connection.dialect.connection_tests}
for test in connection_tests['tests']:
try:
db_validation = self.looker_client.test_connection(connection.name, test)
for item in db_validation:
if item.status != 'success':
db_errors.append("Database Connection {} Test '{}' returned '{}'".format(connection.name,
item.name,
item.message))
except Exception as e:
print("Database Connection test for {} failed due to {}".format(connection.name, e))
if (
connection.name
not in ('looker', 'looker__ilooker', 'looker__internal__analytics')
and connection.dialect
):
connection_tests = {'tests': connection.dialect.connection_tests}
for test in connection_tests['tests']:
try:
db_validation = self.looker_client.test_connection(connection.name, test)
for item in db_validation:
if item.status != 'success':
db_errors.append("Database Connection {} Test '{}' returned '{}'".format(connection.name,
item.name,
item.message))
except Exception as e:
print("Database Connection test for {} failed due to {}".format(connection.name, e))
Comment on lines -22 to +37
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function Connectivity.test_db_connections refactored with the following changes:

return db_errors

def count_all_integrations(self):
Expand All @@ -53,9 +56,6 @@ def test_integrations(self):
integration_errors.append("SUCCESS - Integration {} connectivity test".format(elem.label))
except Exception as e:
integration_errors.append("Integration {} connectivity test could not run due to {}".format(elem.label, e))
else:
# integration_errors.append("Integration {} not enabled.".format(elem.label))
pass
Comment on lines -56 to -58
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function Connectivity.test_integrations refactored with the following changes:

This removes the following comments ( why? ):

# integration_errors.append("Integration {} not enabled.".format(elem.label))

return integration_errors

def count_all_datagroups(self):
Expand All @@ -65,10 +65,10 @@ def count_all_datagroups(self):
def test_datagroups(self):
'''Tests the datagroups and returns failures'''
all_datagroups = self.looker_client.all_datagroups()
group_errors = []
for elem in all_datagroups:
if elem.trigger_error:
group_errors.append("Datagroup \"{}\" on model \"{}\" has this error:\t{}".format(elem.name,
elem.model_name,
elem.trigger_error))
return group_errors
return [
"Datagroup \"{}\" on model \"{}\" has this error:\t{}".format(
elem.name, elem.model_name, elem.trigger_error
)
for elem in all_datagroups
if elem.trigger_error
]
2 changes: 1 addition & 1 deletion modules/content.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def validate_content(self):
'''Runs the Content Validator and returns failures'''
error = self.looker_client.content_validation().content_with_errors
looks_error, dash_errors = [], []
for i in range(0, len(error)):
for i in range(len(error)):
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function Content.validate_content refactored with the following changes:

error_msg = re.sub(r'\[ContentValidationError\(message=', '', str(error[i].errors))
error_msg = re.sub(r'\, field_name=.*', '', error_msg)
if error[i].look:
Expand Down
40 changes: 21 additions & 19 deletions modules/email_content.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
def email_body(looker_url, looker_version, total_users, total_projects, total_erring_content, total_schedules,
total_pdt_errors, pdt_less_30, pdt_30_60, pdt_over_60, unlimited_queries, total_connections, total_integrations, total_datagroups):
'''Generates the summary body for the email'''
body = '''
return '''
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function email_body refactored with the following changes:

Today's Report:<br><br>
Looker instance: {}<br>
Looker version: {}
Expand Down Expand Up @@ -41,19 +41,22 @@ def email_body(looker_url, looker_version, total_users, total_projects, total_er
Find more information <a href="https://github.com/looker-open-source/Themis">go to the repo</a>
<br>
Something Wrong? <a href="https://github.com/looker-open-source/Themis">Tell us</a>
</font>'''.format(looker_version,
looker_url,
total_users,
total_projects,
total_erring_content,
total_schedules,
total_pdt_errors,
pdt_less_30, pdt_30_60, pdt_over_60,
unlimited_queries,
total_connections,
total_integrations,
total_datagroups)
return body
</font>'''.format(
looker_version,
looker_url,
total_users,
total_projects,
total_erring_content,
total_schedules,
total_pdt_errors,
pdt_less_30,
pdt_30_60,
pdt_over_60,
unlimited_queries,
total_connections,
total_integrations,
total_datagroups,
)


def email_attachment(looker_version, looker_url, total_users, user_details, total_projects,
Expand Down Expand Up @@ -85,12 +88,12 @@ def email_attachment(looker_version, looker_url, total_users, user_details, tota
list_errors_schedules = list_errors_schedules,
total_pdt_errors = total_pdt_errors,
list_pdt_errors = list_pdt_errors,

unlimited_queries = unlimited_queries,
report_url = report_url,
is_clustered = is_clustered,
list_nodes = list_nodes,

Comment on lines -88 to +96
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function email_attachment refactored with the following changes:

total_connections = total_connections,
list_errors_connections = list_errors_connections,

Expand All @@ -100,9 +103,8 @@ def email_attachment(looker_version, looker_url, total_users, user_details, tota
total_datagroups = total_datagroups,
list_errors_datagroups = list_errors_datagroups
)
html_file = open('./modules/rendering/rendered_version.html', 'w')
html_file.write(output_text)
html_file.close()
with open('./modules/rendering/rendered_version.html', 'w') as html_file:
html_file.write(output_text)
HTML(filename='./modules/rendering/rendered_version.html').write_pdf('./modules/rendering/final_attachment.pdf',
stylesheets=[CSS(string='@page {{ font-family:arial, serif; font-size: 6; }}')])

Expand Down
17 changes: 7 additions & 10 deletions modules/general.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,25 +17,22 @@ def get_looker_instance():
config = configparser.ConfigParser()
config.read('looker.ini')
config_details = dict(config['Looker'])
instance_url = regex_base_url(config_details['base_url'])
return instance_url
return regex_base_url(config_details['base_url'])
else:
LOOKERSDK_BASE_URL = os.environ.get('LOOKERSDK_BASE_URL')
instance_url = regex_base_url(LOOKERSDK_BASE_URL)
return instance_url
return regex_base_url(LOOKERSDK_BASE_URL)
Comment on lines -20 to +23
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function get_looker_instance refactored with the following changes:


def format_output(function_results):
'''Formats list of errors in Looker to display first 20 elements'''
if isinstance(function_results, (list)):
if len(function_results) >=1:
formatted_results = function_results[:20]
formatted_results.append('...')
return formatted_results
else:
if len(function_results) < 1:
return ['No issues found.']
formatted_results = function_results[:20]
formatted_results.append('...')
return formatted_results
elif isinstance(function_results, (tuple)):
formatted_results = list(function_results)[:20]
formatted_results.append('...')
return formatted_results
return formatted_results
Comment on lines -30 to +36
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function format_output refactored with the following changes:

else:
return function_results
80 changes: 39 additions & 41 deletions modules/performance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,25 +22,25 @@ def unlimited_downloads(self):
"query.limit"
],
filters = {
"history.created_time": "24 hours",
"history.source": "-regenerator,-suggest",
"query.limit": ">5000"
"history.created_time": "24 hours",
"history.source": "-regenerator,-suggest",
"query.limit": ">5000"
},
sorts = ["history.created_time desc"],
limit = "500"
)
unltd_downloads = self.looker_client.create_query(body)
unlimited_downloads = self.looker_client.run_query(unltd_downloads.id, result_format='json')
if unlimited_downloads:
unltd_source, unltd_users = [], []
for unltd_query in json.loads(unlimited_downloads):
unltd_source.append(unltd_query['history.source'])
unltd_users.append(unltd_query['user.id'])
results = "{} users have ran queries with more than 5000 rows from these sources: {}".format(len(list(set(unltd_users))), list(set(unltd_source)))
return results, unltd_downloads.share_url
else:
if not unlimited_downloads:
return None, unltd_downloads.share_url

unltd_source, unltd_users = [], []
for unltd_query in json.loads(unlimited_downloads):
unltd_source.append(unltd_query['history.source'])
unltd_users.append(unltd_query['user.id'])
results = "{} users have ran queries with more than 5000 rows from these sources: {}".format(len(list(set(unltd_users))), list(set(unltd_source)))
return results, unltd_downloads.share_url
Comment on lines -25 to +42
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function Performance.unlimited_downloads refactored with the following changes:


def check_if_clustered(self):
'''Check is Looker is clustered settup'''
body = models.WriteQuery(
Expand All @@ -54,44 +54,42 @@ def check_if_clustered(self):
cluster_check = self.looker_client.create_query(body)
check_clustered = self.looker_client.run_query(cluster_check.id, result_format='json')
nodes_count = len(json.loads(check_clustered))
node_is_cluster = []
for node in json.loads(check_clustered):
node_is_cluster.append(node['node.clustered'])
is_clustered = nodes_count > 1 and list(set(node_is_cluster))[0] == "Yes"
return is_clustered #, len(node_is_cluster)
node_is_cluster = [
node['node.clustered'] for node in json.loads(check_clustered)
]
return nodes_count > 1 and list(set(node_is_cluster))[0] == "Yes"
Comment on lines -57 to +60
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function Performance.check_if_clustered refactored with the following changes:

This removes the following comments ( why? ):

#, len(node_is_cluster)


def nodes_matching(self):
'''For clusters, checks nodes are on same version'''
body = models.WriteQuery(
model = "system__activity",
view = "history",
fields = [
"node.id",
"node.version",
"node.last_heartbeat_time",
"node.last_heartbeat_time"
],
filters = {
"node.last_heartbeat_date": "1 days"
},
sorts = ["node.last_heartbeat_time desc"],
limit = "500",
vis_config = {
"hidden_fields": ["node.id","node.version","node.last_heartbeat_time","most_recent_heartbeat","node.count"]
},
dynamic_fields = "[{\"table_calculation\":\"most_recent_heartbeat\",\"label\":\"most_recent_heartbeat\",\"expression\":\"diff_minutes(${node.last_heartbeat_time}, now())\",\"value_format\":null,\"value_format_name\":null,\"_kind_hint\":\"dimension\",\"_type_hint\":\"number\"},{\"table_calculation\":\"node_version_at_last_beat\",\"label\":\"node_version_at_last_beat\",\"expression\":\"if(diff_minutes(${node.last_heartbeat_time}, now()) > ${most_recent_heartbeat}*1.10 OR diff_minutes(${node.last_heartbeat_time}, now()) < ${most_recent_heartbeat}*0.90, ${node.version}, null)\",\"value_format\":null,\"value_format_name\":null,\"_kind_hint\":\"dimension\",\"_type_hint\":\"string\"}]" )
model = "system__activity",
view = "history",
fields = [
"node.id",
"node.version",
"node.last_heartbeat_time",
"node.last_heartbeat_time"
],
filters = {
"node.last_heartbeat_date": "1 days"
},
sorts = ["node.last_heartbeat_time desc"],
limit = "500",
vis_config = {
"hidden_fields": ["node.id","node.version","node.last_heartbeat_time","most_recent_heartbeat","node.count"]
},
dynamic_fields = "[{\"table_calculation\":\"most_recent_heartbeat\",\"label\":\"most_recent_heartbeat\",\"expression\":\"diff_minutes(${node.last_heartbeat_time}, now())\",\"value_format\":null,\"value_format_name\":null,\"_kind_hint\":\"dimension\",\"_type_hint\":\"number\"},{\"table_calculation\":\"node_version_at_last_beat\",\"label\":\"node_version_at_last_beat\",\"expression\":\"if(diff_minutes(${node.last_heartbeat_time}, now()) > ${most_recent_heartbeat}*1.10 OR diff_minutes(${node.last_heartbeat_time}, now()) < ${most_recent_heartbeat}*0.90, ${node.version}, null)\",\"value_format\":null,\"value_format_name\":null,\"_kind_hint\":\"dimension\",\"_type_hint\":\"string\"}]" )
node_check = self.looker_client.create_query(body)
nodes_versions = self.looker_client.run_query(node_check.id, result_format='json')
results = []
for version in json.loads(nodes_versions):
if version['node_version_at_last_beat']: # to exclude older heartbeat checks with None values
results.append(version['node_version_at_last_beat'])

results = [
version['node_version_at_last_beat']
for version in json.loads(nodes_versions)
if version['node_version_at_last_beat']
]
diff_node_version = []
if len(list(set(results))) == 1:
diff_node_version.append("All {} Nodes found on same Looker version".format(len(results)))
return diff_node_version
else:
else:
for k,v in Counter(results).items():
diff_node_version.append("{} nodes found on version {}".format(v,k))
return diff_node_version
return diff_node_version
12 changes: 2 additions & 10 deletions modules/projects.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,7 @@ def count_all_projects(self):
def all_projects(self):
'''Returns the list of projects'''
my_projects = self.looker_client.all_projects(fields='id,name')
project_ids = []
for i in range(0, len(my_projects)):
project_ids.append(my_projects[i].id)
return project_ids
return [my_projects[i].id for i in range(len(my_projects))]
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function Projects.all_projects refactored with the following changes:


def validate_lookml(self, project_id):
'''Returns LookML validation errors'''
Expand Down Expand Up @@ -48,8 +45,6 @@ def run_git_test(self, project_id):
one_test = self.looker_client.run_git_connection_test(project_id, test_id.id)
if one_test.status != "pass":
git_tests.append("Test ID: {} failed on Project: {}".format(test_id.id, project_id))
else:
pass
Comment on lines -51 to -52
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function Projects.run_git_test refactored with the following changes:

# change session back to 'production'
self.looker_client.update_session({"workspace_id": "production"})
return git_tests
Expand All @@ -75,8 +70,5 @@ def get_lookml_test(self, project_id):
def run_lookml_test(self, project_id):
'''Runs the LookML tests'''
results = self.looker_client.run_lookml_test(project_id)
if not results:
pass
# return "No test set up on Project: {}".format(project_id)
else:
if results:
Comment on lines -78 to +73
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function Projects.run_lookml_test refactored with the following changes:

This removes the following comments ( why? ):

# return "No test set up on Project: {}".format(project_id)

return results
21 changes: 14 additions & 7 deletions modules/schedules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,13 @@ def get_failed_schedules(self):
)
schedules_query = self.looker_client.create_query(body)
failed_schedules = self.looker_client.run_query(schedules_query.id, result_format='json')
cleaned_errors = []
for elem in json.loads(failed_schedules):
cleaned_errors.append("Schedule \'{}\' failed to send to {}".format(elem['scheduled_job.name'], elem['scheduled_plan_destination.type']))
cleaned_errors = [
"Schedule \'{}\' failed to send to {}".format(
elem['scheduled_job.name'], elem['scheduled_plan_destination.type']
)
for elem in json.loads(failed_schedules)
]

Comment on lines -38 to +44
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function Schedules.get_failed_schedules refactored with the following changes:

if failed_schedules:
cleaned_errors = list(set(cleaned_errors)) # set to remove duplicates
return cleaned_errors, len(json.loads(failed_schedules))
Expand All @@ -56,10 +60,13 @@ def get_pdts_status(self):
)
failed_pdts = self.looker_client.create_query(body)
failed_pdts_list = self.looker_client.run_query(failed_pdts.id, result_format='json')
cleaned_errors = []
for elem in json.loads(failed_pdts_list):
# cleaned_errors.append("PDT \'{}\' failed with error: {}".format(elem['pdt_event_log.view_name'], elem['error_message']))
cleaned_errors.append("PDT \'{}\' failed on connection: {}".format(elem['pdt_event_log.view_name'], elem['pdt_event_log.connection']))
cleaned_errors = [
"PDT \'{}\' failed on connection: {}".format(
elem['pdt_event_log.view_name'], elem['pdt_event_log.connection']
)
for elem in json.loads(failed_pdts_list)
]

Comment on lines -59 to +69
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function Schedules.get_pdts_status refactored with the following changes:

This removes the following comments ( why? ):

# cleaned_errors.append("PDT \'{}\' failed with error: {}".format(elem['pdt_event_log.view_name'], elem['error_message']))

if failed_pdts_list:
cleaned_errors = list(set(cleaned_errors)) # set to remove duplicates
# len(json.loads(failed_pdts_list)) will return the number of failures (# of PDTs * build attempts)
Expand Down
4 changes: 1 addition & 3 deletions modules/send_email.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@ def send_report_out(content):
try:
# format env variable like: '[email protected],[email protected]'
email_list = os.environ.get('THEMIS_EMAIL_RECIPIENTS')
to_emails = []
for email in email_list.split(','):
to_emails.append(email)
to_emails = [email for email in email_list.split(',')]
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function send_report_out refactored with the following changes:

except Exception as e:
print("Missing THEMIS_EMAIL_RECIPIENTS Variables {}".format(e))

Expand Down
8 changes: 2 additions & 6 deletions modules/users.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,9 @@ def count_all_users(self):
def get_users_issue(self):
'''Returns locked out users for the Looker instance'''
all_users = self.looker_client.all_users(fields='id, is_disabled')
disabled_users = []
disabled_users = [i for i in all_users if i.is_disabled]

for i in all_users:
if i.is_disabled:
disabled_users.append(i)
user_results = []
user_results.append("Disabled Looker users: {}".format(len(disabled_users)))
user_results = ["Disabled Looker users: {}".format(len(disabled_users))]
Comment on lines -25 to +27
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function Users.get_users_issue refactored with the following changes:

locked_out = self.looker_client.all_user_login_lockouts()
user_results.append("Locked Out Looker users: {}".format(len(locked_out)))
return user_results
Expand Down
Loading