Skip to content

Commit

Permalink
optimize explorer.py
Browse files Browse the repository at this point in the history
  • Loading branch information
@cl117
cl117 committed Aug 15, 2024
1 parent 74c5cd4 commit db907e6
Showing 1 changed file with 103 additions and 120 deletions.
223 changes: 103 additions & 120 deletions flask/explorer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,195 +2,175 @@

from flask import Flask, request, jsonify, abort, render_template
from werkzeug.exceptions import HTTPException

import os
import traceback
import logging
import threading
import time
from flask_debugtoolbar import DebugToolbarExtension
from flask_debugtoolbar_lineprofilerpanel.profile import line_profile

import cluster
import pagerank
import index
import search
import utils
import query

import threading
import time
from flask_debugtoolbar import DebugToolbarExtension
from flask_debugtoolbar_lineprofilerpanel.profile import line_profile


def profile_flask_app():
app.run(debug=True)

if __name__ == "__main__":
#profiler = profile.Profile()
#profiler.enable()
profile_flask_app()
#profiler.disable()
#profiler.print_stats(sort='time')

# Configure logging, This will affect all loggers in your application, not just the Werkzeug logger.
log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)

app = Flask(__name__)
app.config['SECRET_KEY'] = 'your-secret-key' # Required for the debug toolbar
app.config['DEBUG'] = True
app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
# Profiler configuration
app.config['DEBUG_TB_PROFILER_ENABLED'] = True
app.config['DEBUG_TB_PANELS'] = [
'flask_debugtoolbar.panels.versions.VersionDebugPanel',
'flask_debugtoolbar.panels.timer.TimerDebugPanel',
'flask_debugtoolbar.panels.headers.HeaderDebugPanel',
'flask_debugtoolbar.panels.request_vars.RequestVarsDebugPanel',
'flask_debugtoolbar.panels.config_vars.ConfigVarsDebugPanel',
'flask_debugtoolbar.panels.template.TemplateDebugPanel',
'flask_debugtoolbar.panels.logger.LoggingPanel',
'flask_debugtoolbar.panels.profiler.ProfilerDebugPanel',
'flask_debugtoolbar_lineprofilerpanel.panels.LineProfilerPanel'
]
app.config.update(
SECRET_KEY='your-secret-key', # Required for the debug toolbar
DEBUG=True,
DEBUG_TB_INTERCEPT_REDIRECTS=False,
DEBUG_TB_PROFILER_ENABLED=True,
DEBUG_TB_PANELS=[
'flask_debugtoolbar.panels.versions.VersionDebugPanel',
'flask_debugtoolbar.panels.timer.TimerDebugPanel',
'flask_debugtoolbar.panels.headers.HeaderDebugPanel',
'flask_debugtoolbar.panels.request_vars.RequestVarsDebugPanel',
'flask_debugtoolbar.panels.config_vars.ConfigVarsDebugPanel',
'flask_debugtoolbar.panels.template.TemplateDebugPanel',
'flask_debugtoolbar.panels.logger.LoggingPanel',
'flask_debugtoolbar.panels.profiler.ProfilerDebugPanel',
'flask_debugtoolbar_lineprofilerpanel.panels.LineProfilerPanel'
]
)

# Initialize the debug toolbar
toolbar = DebugToolbarExtension(app)


# Error handler
@app.errorhandler(Exception)
def handle_error(e):
utils.log('[ERROR] Returning error ' + str(e) + "\n Traceback:\n" + traceback.format_exc())

log.error(f'[ERROR] Returning error {e}\n Traceback:\n{traceback.format_exc()}')
if isinstance(e, HTTPException):
return jsonify(error=str(e.name + ": " + e.description)), e.code
else:
return jsonify(error=str(type(e).__name__) + str(e)), 500
return jsonify(error=str(type(e).__name__) + str(e)), 500

@app.before_first_request
def startup():
# Method for running auto indexing
def auto_update_index():
update_interval = int(utils.get_config().get('updateTimeInDays', 0)) * 86400
while True:
time.sleep(int(utils.get_config()['updateTimeInDays']) * 86400)
# if utils.get_config()['autoUpdateIndex'] and utils.get_config()['updateTimeInDays'] > 0:
# utils.log('Updating index automatically. To disable, set the \"autoUpdateIndex\" property in config.json to false.')
# update()
time.sleep(update_interval)
# Implement your update logic here
if utils.get_config().get('autoUpdateIndex', False):
update_index()

# Thread for automatically updaing the index periodically
# Start the background thread for auto-updating the index
update_thread = threading.Thread(target=auto_update_index, daemon=True)
update_thread.start()

if os.path.exists('log.txt') and os.path.getsize('log.txt') > 20000000: # Delete the log if it is > 20 MB
os.remove('log.txt')

if os.path.exists('indexing_log.txt') and os.path.getsize('indexing_log.txt') > 20000000: # Delete the log if it is > 20 MB
os.remove('indexing_log.txt')
# Manage log file sizes
for log_file in ['log.txt', 'indexing_log.txt']:
if os.path.exists(log_file) and os.path.getsize(log_file) > 20000000: # 20 MB
os.remove(log_file)

utils.log('SBOLExplorer started :)')

# Check and create index if necessary
try:
if utils.get_es().indices.exists(index=utils.get_config()['elasticsearch_index_name']) is False:
es = utils.get_es()
index_name = utils.get_config().get('elasticsearch_index_name')
if not es.indices.exists(index=index_name):
utils.log('Index not found, creating new index.')
update()
except:
update_index()
except Exception as e:
log.error(f'Error during startup: {e}')
raise

@app.errorhandler(Exception)
def handle_error(e):
utils.log('[ERROR] Returning error ' + str(e) + "\n Traceback:\n" + traceback.format_exc())
return jsonify(error=str(e)), 500
def update_index():
utils.log_indexing('============ STARTING INDEXING ============\n\n')
utils.log('============ STARTING INDEXING ============\n\n')
utils.save_update_start_time()

clusters = cluster.update_clusters()
utils.save_clusters(clusters)

uri2rank = pagerank.update_pagerank()
utils.save_uri2rank(uri2rank)

index.update_index(utils.get_uri2rank())

query.memoized_query_sparql.cache_clear()
utils.log_indexing('Cache cleared')

utils.save_update_end_time()
utils.log_indexing('============ INDEXING COMPLETED ============\n\n')
utils.log('============ INDEXING COMPLETED ============\n\n')

@app.route('/info', methods=['GET'])
def info():
utils.log('Explorer up!!! Virtutoso ' + str(query.memoized_query_sparql.cache_info()))
utils.log('Explorer up!!! Virtuoso ' + str(query.memoized_query_sparql.cache_info()))
return utils.get_log()

@app.route('/indexinginfo', methods=['GET'])
def indexinginfo():
return utils.get_indexing_log()

@app.route('/config', methods=['POST', 'GET'])
def config():
def config_route():
if request.method == 'POST':
new_config = request.get_json()
utils.set_config(new_config)
utils.log('Successfully updated config')

config = utils.get_config()
return jsonify(config)

return jsonify(utils.get_config())

@app.route('/update', methods=['GET'])
def update():
try:
subject = request.args.get('subject')

if subject is None:
utils.log_indexing('============ STARTING INDEXING ============\n\n')
utils.log('============ STARTING INDEXING ============\n\n')
utils.save_update_start_time()

clusters = cluster.update_clusters()
utils.save_clusters(clusters)


uri2rank = pagerank.update_pagerank()
utils.save_uri2rank(uri2rank)

index.update_index(utils.get_uri2rank())

query.memoized_query_sparql.cache_clear()
utils.log_indexing('Cache cleared')

utils.save_update_end_time()
success_message = 'Successfully updated entire index'
else:
if subject:
index.refresh_index(subject, utils.get_uri2rank())
success_message = 'Successfully refreshed: ' + subject

utils.log_indexing('============ INDEXING COMPLETED ============\n\n')
utils.log('============ INDEXING COMPLETED ============\n\n')
success_message = f'Successfully refreshed: {subject}'
else:
update_index()
success_message = 'Successfully updated entire index'
return success_message
except Exception as e:
utils.log_indexing('[ERROR] Returning error ' + str(e) + "\n Traceback:\n" + traceback.format_exc())

log.error(f'Error during update: {e}')
raise

@app.route('/incrementalupdate', methods=['POST'])
def incremental_update():
try:
updates = request.get_json()

index.incremental_update(updates, utils.get_uri2rank())

success_message = 'Successfully incrementally updated parts'
utils.log(success_message)
return
except:
return success_message
except Exception as e:
log.error(f'Error during incremental update: {e}')
raise


@app.route('/incrementalremove', methods=['GET'])
def incremental_remove():
try:
subject = request.args.get('subject')

index.incremental_remove(subject)

success_message = 'Successfully incrementally removed: ' + subject
success_message = f'Successfully incrementally removed: {subject}'
utils.log(success_message)
return success_message
except:
except Exception as e:
log.error(f'Error during incremental remove: {e}')
raise

@app.route('/incrementalremovecollection', methods=['GET'])
def incremental_remove_collection():
try:
subject = request.args.get('subject')
uri_prefix = request.args.get('uriPrefix')

index.incremental_remove_collection(subject, uri_prefix)

success_message = 'Successfully incrementally removed collection and members: ' + subject
success_message = f'Successfully incrementally removed collection and members: {subject}'
utils.log(success_message)
return success_message
except:
except Exception as e:
log.error(f'Error during incremental remove collection: {e}')
raise

@app.route('/test', methods=['GET'])
Expand All @@ -202,44 +182,47 @@ def SBOLExplore_test_endpoint():
@line_profile
def sparql_search_endpoint():
try:
# make sure index is built, or throw exception
if utils.get_es().indices.exists(index=utils.get_config()['elasticsearch_index_name']) is False or utils.get_es().cat.indices(format='json')[0]['health'] is 'red':
es = utils.get_es()
index_name = utils.get_config().get('elasticsearch_index_name')
if not es.indices.exists(index=index_name) or es.cat.indices(format='json')[0]['health'] == 'red':
abort(503, 'Elasticsearch is not working or the index does not exist.')

sparql_query = request.args.get('query')

if sparql_query is not None:
if sparql_query:
default_graph_uri = request.args.get('default-graph-uri')
response = jsonify(
search.search(
sparql_query,
utils.get_uri2rank(),
utils.get_clusters(),
default_graph_uri
))
response = jsonify(search.search(
sparql_query,
utils.get_uri2rank(),
utils.get_clusters(),
default_graph_uri
))
return response
else:
return "<pre><h1>Welcome to SBOLExplorer! <br> <h2>The available indices in Elasticsearch are shown below:</h2></h1><br>"\
return "<pre><h1>Welcome to SBOLExplorer! <br> <h2>The available indices in Elasticsearch are shown below:</h2></h1><br>"\
+ str(utils.get_es().cat.indices(format='json'))\
+ "<br><br><h3>The config options are set to:</h3><br>"\
+ str(utils.get_config())\
+ "<br><br><br><br><a href=\"https://github.com/synbiodex/sbolexplorer\">Visit our GitHub repository!</a>"\
+ "<br><br>Any issues can be reported to our <a href=\"https://github.com/synbiodex/sbolexplorer/issues\">issue tracker.</a>"\
+ "<br><br>Used by <a href=\"https://github.com/synbiohub/synbiohub\">SynBioHub.</a>"
#return render_template('index.html')
except:

except Exception as e:
log.error(f'Error during SPARQL search: {e}')
raise

@app.route('/search', methods=['GET'])
def search_by_string():
try:
if utils.get_es().indices.exists(index=utils.get_config()['elasticsearch_index_name']) is False or utils.get_es().cat.indices(format='json')[0]['health'] is 'red':
es = utils.get_es()
index_name = utils.get_config().get('elasticsearch_index_name')
if not es.indices.exists(index=index_name) or es.cat.indices(format='json')[0]['health'] == 'red':
abort(503, 'Elasticsearch is not working or the index does not exist.')

query = request.args.get('query')

response = jsonify(search.search_es(query)['hits'])

return response
except:
except Exception as e:
log.error(f'Error during search by string: {e}')
raise

if __name__ == "__main__":
app.run(debug=True)

0 comments on commit db907e6

Please sign in to comment.