From d7378284300498e17424830f8322a5d7dd304fb0 Mon Sep 17 00:00:00 2001 From: Edwin Huang <43209173+edwin5588@users.noreply.github.com> Date: Fri, 27 Sep 2024 12:24:34 -0700 Subject: [PATCH 01/11] updated unzipping function for aggregator --- caper/caper/aggregator_main.py | 32 +++++++++++++++++-------- caper/templates/pages/edit_project.html | 3 +-- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/caper/caper/aggregator_main.py b/caper/caper/aggregator_main.py index 073b25fc..aed8daa4 100644 --- a/caper/caper/aggregator_main.py +++ b/caper/caper/aggregator_main.py @@ -63,24 +63,36 @@ def read_name_remap(name_remap_file): def unzip_file(fp, dest_root): """ - unzips file based on zip type + Unzips file based on zip type. + Ensures proper extraction of all files, including nested directories. """ try: + print('hello') if fp.endswith(".tar.gz"): zip_name = os.path.basename(fp).replace(".tar.gz", "") - destination = f'{dest_root}/{zip_name}' - with tarfile.open(fp, 'r') as output_zip: - output_zip.extractall(destination) - output_zip.close() + destination = os.path.join(dest_root, zip_name) + os.makedirs(destination, exist_ok=True) # Ensure destination exists + + # Open and extract tar.gz + with tarfile.open(fp, 'r:gz') as tar_ref: + for member in tar_ref.getmembers(): + if member.isreg(): + member.name = os.path.basename(member.name) + tar_ref.extract(member, destination) + elif fp.endswith(".zip"): zip_name = os.path.basename(fp).replace(".zip", "") - destination = f'{dest_root}/{zip_name}' + destination = os.path.join(dest_root, zip_name) + os.makedirs(destination, exist_ok=True) # Ensure destination exists + + # Open and extract zip with zipfile.ZipFile(fp, 'r') as zip_ref: zip_ref.extractall(destination) - zip_ref.close() + + print(f'Just extracted: {fp} to {destination}!') except Exception as e: - print(e) + print(f"Error occurred while extracting {fp}: {e}") def clean_dirs(dlist): @@ -135,6 +147,7 @@ def unzip(self): Unzips the zip files, and get directories for files within """ + print('hello im here') for zip_fp in self.zip_paths: fp = os.path.join(self.root, zip_fp) try: @@ -162,6 +175,7 @@ def unzip(self): ## find samples and move files # samples = [] aa_samples_found = 0 + print(f'*************************************************{self.DEST_ROOT}') print("Crawling files for AA, classification, and CN data...") for root, dirs, files in os.walk(self.DEST_ROOT, topdown = True): for dir in dirs: @@ -365,7 +379,6 @@ def cleanup(self): """ Zips the aggregate results, and deletes files for cleanup """ - print(self.samp_AA_dct.values()) clean_dirs(self.samp_AA_dct.values()) # self.clean_files(self.samp_ckit_dct.values()) print("Creating tar.gz...") @@ -373,7 +386,6 @@ def cleanup(self): self.tardir(f'{self.ROOT_FP}/results', f'{self.output_name}.tar.gz') print('cleaning directories now ... ') clean_dirs([f'{self.ROOT_FP}/results']) # ./extracted_from_zips - # def find_file(self, basename): diff --git a/caper/templates/pages/edit_project.html b/caper/templates/pages/edit_project.html index ec5d2f80..fa7e8c5e 100644 --- a/caper/templates/pages/edit_project.html +++ b/caper/templates/pages/edit_project.html @@ -270,8 +270,7 @@

Please sign in to edit projects

//xhr.send(fd); } - - + document.addEventListener('DOMContentLoaded', function() { // Pass the alias names from Django context to JavaScript var aliasNames = JSON.parse("{{ all_alias | escapejs}}"); From 3d43c9613df2a0584578caf580f954e5fc645e44 Mon Sep 17 00:00:00 2001 From: Edwin Huang <43209173+edwin5588@users.noreply.github.com> Date: Fri, 27 Sep 2024 15:40:17 -0700 Subject: [PATCH 02/11] update fix for merge --- caper/caper/aggregator_main.py | 8 ++------ caper/caper/views.py | 4 ---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/caper/caper/aggregator_main.py b/caper/caper/aggregator_main.py index aed8daa4..90345785 100644 --- a/caper/caper/aggregator_main.py +++ b/caper/caper/aggregator_main.py @@ -67,12 +67,10 @@ def unzip_file(fp, dest_root): Ensures proper extraction of all files, including nested directories. """ try: - print('hello') if fp.endswith(".tar.gz"): zip_name = os.path.basename(fp).replace(".tar.gz", "") destination = os.path.join(dest_root, zip_name) os.makedirs(destination, exist_ok=True) # Ensure destination exists - # Open and extract tar.gz with tarfile.open(fp, 'r:gz') as tar_ref: for member in tar_ref.getmembers(): @@ -84,7 +82,6 @@ def unzip_file(fp, dest_root): zip_name = os.path.basename(fp).replace(".zip", "") destination = os.path.join(dest_root, zip_name) os.makedirs(destination, exist_ok=True) # Ensure destination exists - # Open and extract zip with zipfile.ZipFile(fp, 'r') as zip_ref: zip_ref.extractall(destination) @@ -147,7 +144,6 @@ def unzip(self): Unzips the zip files, and get directories for files within """ - print('hello im here') for zip_fp in self.zip_paths: fp = os.path.join(self.root, zip_fp) try: @@ -175,7 +171,6 @@ def unzip(self): ## find samples and move files # samples = [] aa_samples_found = 0 - print(f'*************************************************{self.DEST_ROOT}') print("Crawling files for AA, classification, and CN data...") for root, dirs, files in os.walk(self.DEST_ROOT, topdown = True): for dir in dirs: @@ -472,6 +467,8 @@ def json_modifications(self): if feature in sample_dct and sample_dct[feature]: feat_basename = os.path.basename(sample_dct[feature]) feat_file = f'{self.sample_to_ac_location_dct[sample]}/files/{feat_basename}' + if not os.path.exists(feat_file): + feat_file = f'{self.sample_to_ac_location_dct[sample]}/{feat_basename}' if feature == "CNV BED file" and any([feat_file.endswith(x) for x in ["AA_CNV_SEEDS.bed", "CNV_CALLS_pre_filtered.bed", "Not provided", "Not Provided"]]): cnvkit_dir = self.samp_ckit_dct[sample_dct['Sample name']] if cnvkit_dir: @@ -551,7 +548,6 @@ def clean_by_suffix(self, suffix, dir): # except FileNotFoundError: # pass - # TODO: VALIDATE IS NEVER USED! def validate(): """ diff --git a/caper/caper/views.py b/caper/caper/views.py index c2286d28..d326ccea 100644 --- a/caper/caper/views.py +++ b/caper/caper/views.py @@ -1316,7 +1316,6 @@ def edit_project_page(request, project_name): ## build download URL url = f'http://127.0.0.1:8000/project/{project["linkid"]}/download' download_path = project_data_path+'/download.tar.gz' - try: ## try to download old project file download = download_file(url, download_path) @@ -1716,7 +1715,6 @@ def extract_project_files(tarfile, file_location, project_data_path, project_id) for feature in features: # logging.debug(feature['Sample name']) if len(feature) > 0: - # get paths key_names = ['Feature BED file', 'CNV BED file', 'AA PDF file', 'AA PNG file', 'Sample metadata JSON', 'AA directory', 'cnvkit directory'] @@ -1725,10 +1723,8 @@ def extract_project_files(tarfile, file_location, project_data_path, project_id) path_var = feature[k] with open(f'{project_data_path}/results/{path_var}', "rb") as file_var: id_var = fs_handle.put(file_var) - except: id_var = "Not Provided" - feature[k] = id_var # Now update the project with the updated runs From d69537c7462b787d4fd49e2a1428b8d84b7bc0da Mon Sep 17 00:00:00 2001 From: Edwin Huang <43209173+edwin5588@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:18:11 -0700 Subject: [PATCH 03/11] getting rid of print statement --- caper/caper/views.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/caper/caper/views.py b/caper/caper/views.py index d326ccea..7ec27778 100644 --- a/caper/caper/views.py +++ b/caper/caper/views.py @@ -1423,9 +1423,6 @@ def edit_project_page(request, project_name): memberString = ', '.join(members) form = UpdateForm(initial={"project_name": project['project_name'],"description": project['description'],"private":project['private'],"project_members": memberString,"publication_link": publication_link}) - - - print(project['alias_name']) return render(request, "pages/edit_project.html", {'project': project, 'run': form, From 25ab600a27fa47ecac3fbe215611627fc021873b Mon Sep 17 00:00:00 2001 From: Edwin Huang <43209173+edwin5588@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:56:53 -0700 Subject: [PATCH 04/11] alias logic fixed for editing project without uploading file --- caper/caper/views.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/caper/caper/views.py b/caper/caper/views.py index 7ec27778..868427fb 100644 --- a/caper/caper/views.py +++ b/caper/caper/views.py @@ -1392,9 +1392,14 @@ def edit_project_page(request, project_name): if runs != 0: current_runs.update(runs) query = {'_id': ObjectId(project_name)} + try: + alias_name = form_dict['alias'] + print(alias_name) + except: + print('no alias to be found') new_val = { "$set": {'project_name':new_project_name, 'runs' : current_runs, 'description': form_dict['description'], 'date': get_date(), 'private': form_dict['private'], 'project_members': form_dict['project_members'], 'publication_link': form_dict['publication_link'], - 'Oncogenes': get_project_oncogenes(current_runs)}} + 'Oncogenes': get_project_oncogenes(current_runs), 'alias_name' : alias_name}} if form.is_valid(): print('im here') collection_handle.update_one(query, new_val) From 2b2a3ce2be4ff7439306b5ba942fd23d5bc40d9d Mon Sep 17 00:00:00 2001 From: liefeld Date: Wed, 2 Oct 2024 20:51:01 +0000 Subject: [PATCH 05/11] updating client ids --- caper/caper/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/caper/caper/settings.py b/caper/caper/settings.py index 9c3a9e35..619cf5a7 100644 --- a/caper/caper/settings.py +++ b/caper/caper/settings.py @@ -189,7 +189,8 @@ ], 'AUTH_PARAMS': {'access_type': 'online'}, 'APP': { - 'client_id': '789453891819-hk9q466oq5ba8i2ur4pk8d0of2f056sc.apps.googleusercontent.com', + #'client_id': '789453891819-hk9q466oq5ba8i2ur4pk8d0of2f056sc.apps.googleusercontent.com', + 'client_id': '715102420712-3c11l0918iers60ca8eifnnpuihu88sm.apps.googleusercontent.com', 'secret': GOOGLE_SECRET_KEY, } }, @@ -204,7 +205,7 @@ 'urn:globus:auth:scope:transfer.api.globus.org:all' ], 'APP': { - 'client_id': '6524a9d1-a235-4c4f-9e20-c70e77f6e34b', + 'client_id': '61af67c2-8697-4afd-a5df-2a46a8ef17df', 'secret': GLOBUS_SECRET_KEY, } } From 4c231745d1684ef03088fde84cb78ce8a6a96512 Mon Sep 17 00:00:00 2001 From: Edwin Huang <43209173+edwin5588@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:18:32 -0700 Subject: [PATCH 06/11] fixes for... 1. added switch to replace / append to project 2. error handling for aggregator so it doesn't go to "Sorry, error occurred" page. --- caper/caper/aggregator_main.py | 13 +++++++++-- caper/caper/forms.py | 9 +++++++ caper/caper/views.py | 31 ++++++++++++++++++------- caper/templates/pages/edit_project.html | 1 - 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/caper/caper/aggregator_main.py b/caper/caper/aggregator_main.py index 90345785..1c0e585f 100644 --- a/caper/caper/aggregator_main.py +++ b/caper/caper/aggregator_main.py @@ -136,8 +136,13 @@ def __init__(self, filelist, output_name, run_classifier, ref, py3_path, output_ self.samp_mdata_dct, self.run_mdata_dct = defaultdict(str), defaultdict(str) self.locate_dirs_and_metadata_jsons() self.sample_to_ac_location_dct = self.aggregate_tables() - self.json_modifications() + complete = self.json_modifications() self.cleanup() + + if complete: + self.complete = True + else: + self.complete = False def unzip(self): """ @@ -420,7 +425,9 @@ def json_modifications(self): sys.stderr.write(str(ref_genomes) + "\n") sys.stderr.write("ERROR! Multiple reference genomes detected in project.\n AmpliconRepository only " "supports single-reference projects currently. Exiting.\n") - sys.exit(1) + # sys.exit(1) + return None + potential_str_lsts = [ 'Location', @@ -536,6 +543,8 @@ def json_modifications(self): aggregate = pd.DataFrame.from_records(flattened_samples) aggregate.to_csv(f'{self.ROOT_FP}/results/aggregated_results.csv') aggregate.to_html(f'{self.ROOT_FP}/results/aggregated_results.html') + + return 'Completed json mods' def clean_by_suffix(self, suffix, dir): if suffix and dir and not dir == "/" and not suffix == "*": diff --git a/caper/caper/forms.py b/caper/caper/forms.py index fa8ecdf3..58d2d2a9 100644 --- a/caper/caper/forms.py +++ b/caper/caper/forms.py @@ -48,6 +48,15 @@ class UpdateForm(forms.ModelForm): help_text= 'Click checkbox to acknowledge and accept the terms of the license agreement.', ) + + replace_project = forms.BooleanField( + label = format_html( + "Replace Project? If ticked, will replace the entire project with the file you upload." + ), + required = False, + widget = forms.CheckboxInput(), + help_text = 'Click Checkbox to replace project with a new project.' + ) class Meta: model = Run diff --git a/caper/caper/views.py b/caper/caper/views.py index 868427fb..ea7aadb7 100644 --- a/caper/caper/views.py +++ b/caper/caper/views.py @@ -1282,16 +1282,13 @@ def download_file(url, save_path): def edit_project_page(request, project_name): if request.method == "POST": project = get_one_project(project_name) - # no edits for non-project members if not is_user_a_project_member(project, request): return HttpResponse("Project does not exist") form = UpdateForm(request.POST, request.FILES) form_dict = form_to_dict(form) - form_dict['project_members'] = create_user_list(form_dict['project_members'], get_current_user(request)) - # lets notify users (if their preferences request it) if project membership has changed new_membership = form_dict['project_members'] old_membership = project['project_members'] @@ -1318,13 +1315,25 @@ def edit_project_page(request, project_name): download_path = project_data_path+'/download.tar.gz' try: ## try to download old project file - download = download_file(url, download_path) print(f"PREVIOUS FILE FPS LIST: {file_fps}") - file_fps.append(os.path.join('download.tar.gz')) + ### if replace project, don't download old project + try: + if request.POST['replace_project'] == 'on': + print('Replacing project with new uploaded file') + except: + download = download_file(url, download_path) + file_fps.append(os.path.join('download.tar.gz')) print(f"AFTERS FILE FPS LIST: {file_fps}") - print(f'aggregating on: {file_fps}') agg = Aggregator(file_fps, project_data_path, 'No', "", 'python3', output_directory = f'{temp_proj_id}') + if agg.complete != True: + ## redirect to edit page if aggregator fails + alert_message = "Edit project failed. Please ensure all uploaded samples have the same reference genome and are valid AmplionSuite results." + return render(request, 'pages/edit_project.html', + {'project': project, + 'run': form, + 'alert_message': alert_message, + 'all_alias' :get_all_alias()}) ## after running aggregator, replace the requests file with the aggregated file: with open(agg.aggregated_filename, 'rb') as f: uploaded_file = SimpleUploadedFile( @@ -1376,7 +1385,7 @@ def edit_project_page(request, project_name): {'project': project, 'run': form, 'alert_message': alert_message, - 'all_alias' :get_all_alias()}) + 'all_alias' :json.dumps(get_all_alias())}) # JTL 081823 Not sure what these next 4 lines are about? An earlier plan to change the project file? # leaving them alone for now but they smell like dead code if 'file' in form_dict: @@ -1927,7 +1936,13 @@ def create_project(request): file_fps.append(file.name) file.close() agg = Aggregator(file_fps, project_data_path, 'No', "", 'python3', output_directory = f'{temp_proj_id}') - + if agg.complete != True: + ## redirect to edit page if aggregator fails + alert_message = "Create project failed. Please ensure all uploaded samples have the same reference genome and are valid AmplionSuite results." + return render(request, 'pages/create_project.html', + {'run': form, + 'alert_message': alert_message, + 'all_alias':json.dumps(get_all_alias())}) ## after running aggregator, replace the requests file with the aggregated file: with open(agg.aggregated_filename, 'rb') as f: uploaded_file = SimpleUploadedFile( diff --git a/caper/templates/pages/edit_project.html b/caper/templates/pages/edit_project.html index fa7e8c5e..85755c68 100644 --- a/caper/templates/pages/edit_project.html +++ b/caper/templates/pages/edit_project.html @@ -78,7 +78,6 @@

Editing project: '{{ project.project_name }}'


{% else %} {{field | as_crispy_field }} - {% endif %} {% endfor %}
From 1c39be523a05bd025afa186cb1d564026087d15c Mon Sep 17 00:00:00 2001 From: liefeld Date: Mon, 21 Oct 2024 18:12:59 +0000 Subject: [PATCH 07/11] try --- caper/templates/pages/create_project.html | 213 +++++++++++----------- 1 file changed, 105 insertions(+), 108 deletions(-) diff --git a/caper/templates/pages/create_project.html b/caper/templates/pages/create_project.html index 7f0ddd79..db1ef372 100644 --- a/caper/templates/pages/create_project.html +++ b/caper/templates/pages/create_project.html @@ -174,171 +174,167 @@

Create New Project

{% endif %} {% endblock main %} \ No newline at end of file From 044ba95a39a5dfd5ef98d5933cfeb4917a13ab7b Mon Sep 17 00:00:00 2001 From: liefeld Date: Mon, 21 Oct 2024 20:02:12 +0000 Subject: [PATCH 08/11] edwin's dev commits for CSS tweaks & feature updates --- caper/caper/forms.py | 1 + caper/caper/views.py | 29 ++++-- caper/templates/pages/edit_project.html | 120 ++++++++++++++++++------ 3 files changed, 114 insertions(+), 36 deletions(-) diff --git a/caper/caper/forms.py b/caper/caper/forms.py index 58d2d2a9..23e5b008 100644 --- a/caper/caper/forms.py +++ b/caper/caper/forms.py @@ -77,6 +77,7 @@ def __init__(self, *args, **kwargs): self.fields['publication_link'].widget.attrs.update({'placeholder': 'Optional: Provide a PMID or link to a publication'}) self.fields['project_members'].required = False self.fields['project_members'].widget.attrs.update({'placeholder': 'Optional: List of additional email addresses or AmpliconRepository usernames separated by spaces or commas'}) + self.fields['replace_project'].widget.attrs.update({'id': 'custom_id_replace_project'}) # self.fields['file'].required = False diff --git a/caper/caper/views.py b/caper/caper/views.py index ea7aadb7..c25197a5 100644 --- a/caper/caper/views.py +++ b/caper/caper/views.py @@ -559,8 +559,6 @@ def project_page(request, project_name, message=''): ## if flag is unfinished, render a loading page: project = validate_project(get_one_project(project_name), project_name) - print(project.keys()) - print(project['private']) if 'FINISHED?' in project and project['FINISHED?'] == False: return render(request, "pages/loading.html", {"project_name":project_name}) @@ -1277,17 +1275,38 @@ def download_file(url, save_path): print(f"File downloaded successfully and saved to {save_path}") - + def edit_project_page(request, project_name): + if request.method == "POST": project = get_one_project(project_name) + old_alias_name = None + if 'alias_name' in project: + + old_alias_name = project['alias_name'] + print(f'THE OLD ALIAS NAME SHOULD BE: {old_alias_name}') # no edits for non-project members if not is_user_a_project_member(project, request): return HttpResponse("Project does not exist") form = UpdateForm(request.POST, request.FILES) + ## give the new project the old project alias. + if form.data['alias'] == '': + if old_alias_name: + mutable_data = form.data.copy() # Make a mutable copy of the form's data + mutable_data['alias'] = old_alias_name # Set the alias to the new value + form.data = mutable_data + ## update old project so its alias is set to None, and the alias is set to the new project + query = {'_id': ObjectId(project_name)} + new_val = { "$set": {'alias_name' : None}} + collection_handle.update_one(query, new_val) + form_dict = form_to_dict(form) + print('UPDATED FORM ALIAS') + print(form_dict['alias']) + print(form.data) + form_dict['project_members'] = create_user_list(form_dict['project_members'], get_current_user(request)) # lets notify users (if their preferences request it) if project membership has changed new_membership = form_dict['project_members'] @@ -1406,6 +1425,7 @@ def edit_project_page(request, project_name): print(alias_name) except: print('no alias to be found') + new_val = { "$set": {'project_name':new_project_name, 'runs' : current_runs, 'description': form_dict['description'], 'date': get_date(), 'private': form_dict['private'], 'project_members': form_dict['project_members'], 'publication_link': form_dict['publication_link'], 'Oncogenes': get_project_oncogenes(current_runs), 'alias_name' : alias_name}} @@ -1417,8 +1437,6 @@ def edit_project_page(request, project_name): raise Http404() else: return HttpResponse("Project does not exist") - - else: project = get_one_project(project_name) prev_versions, prev_ver_msg = previous_versions(project) @@ -1426,7 +1444,6 @@ def edit_project_page(request, project_name): messages.error(request, "Redirected to latest version, editing of old versions not allowed. ") return redirect('project_page', project_name = prev_versions[0]['linkid']) - print(prev_ver_msg) # split up the project members and remove the empties members = project['project_members'] try: diff --git a/caper/templates/pages/edit_project.html b/caper/templates/pages/edit_project.html index 85755c68..a017b6c6 100644 --- a/caper/templates/pages/edit_project.html +++ b/caper/templates/pages/edit_project.html @@ -62,6 +62,8 @@

Editing project: '{{ project.project_name }}'

{% for field in run %} {% if field.name == 'alias' %} +
+

Enter an unique alias for this project. It can be used to access this project.

{% if project.alias_name != None %} @@ -71,53 +73,75 @@

Editing project: '{{ project.project_name }}'

ampliconrepository.org/project/ -
{{ field }}
+
{{ field }}
/
+
+
{% else %} - {{field | as_crispy_field }} + {% if field.name != 'accept_license' and field.name != 'replace_project' %} + {{field | as_crispy_field }} + {% endif %} + {% endif %} {% endfor %} -
-
- - To add samples to the project, drag files here or -
- - select files + +
+
+
+
+ +
+
+
+
+ {{ run.replace_project | as_crispy_field }} +
+
+ + To add samples to the project, drag files here or +
+ + select files +
+
+
+ + + + + + + + + + + + + + +
File NameFile SizeRemove
+
+
- - - - - - - - - - - - - - -
File NameFile SizeRemove
- -
-
- + - +
+ {{ run.accept_license | as_crispy_field }} + - + @@ -292,6 +316,42 @@

Please sign in to edit projects

}); + $(document).ready(function() { + // Get references to the checkbox, file input, and submit button + const $checkbox = $('#custom_id_replace_project'); + const $fileInput = $('#fileuploadfield'); + const $submitButton = $('#createProjectBtn'); + + // Function to toggle the submit button based on checkbox and file input status + function toggleSubmitButton() { + const isChecked = $checkbox.is(':checked'); // Check if checkbox is checked + const filesUploaded = $fileInput[0].files.length; // Get the number of uploaded files + + console.log('Checkbox checked:', isChecked); + console.log('Files uploaded:', filesUploaded); + + if (isChecked && filesUploaded === 0) { + $submitButton.prop('disabled', true); // Disable submit button + console.log('Submit button disabled'); + } else { + $submitButton.prop('disabled', false); // Enable submit button + console.log('Submit button enabled'); + } + } + + // Event listener for checkbox change + $checkbox.change(toggleSubmitButton); + + // Event listener for file input change + $fileInput.change(toggleSubmitButton); +}); + + + + + + + From 0807a418b5915d7e2bb9e1805c41d44823816ec7 Mon Sep 17 00:00:00 2001 From: liefeld Date: Mon, 21 Oct 2024 20:13:00 +0000 Subject: [PATCH 09/11] edwin's DEV changes for css tweak pt2 --- caper/caper/forms.py | 2 +- caper/templates/pages/create_project.html | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/caper/caper/forms.py b/caper/caper/forms.py index 23e5b008..eb6b4dc2 100644 --- a/caper/caper/forms.py +++ b/caper/caper/forms.py @@ -55,7 +55,7 @@ class UpdateForm(forms.ModelForm): ), required = False, widget = forms.CheckboxInput(), - help_text = 'Click Checkbox to replace project with a new project.' + help_text = 'The default behavior is to add samples to the current project.' ) class Meta: diff --git a/caper/templates/pages/create_project.html b/caper/templates/pages/create_project.html index db1ef372..771ef7b0 100644 --- a/caper/templates/pages/create_project.html +++ b/caper/templates/pages/create_project.html @@ -67,6 +67,7 @@

Create New Project

{% for field in run %} {% if field.name == 'alias' %} +

Enter an unique alias for this project. It can be used to access this project.

ampliconrepository.org/project/ @@ -76,10 +77,11 @@

Create New Project



- +
{% else %} + {% if field.name != 'accept_license' %} {{field | as_crispy_field }} - + {% endif %} {% endif %} {% endfor %}
@@ -116,6 +118,10 @@

Create New Project

+
+ {{run.accept_license | as_crispy_field }} + + Date: Wed, 23 Oct 2024 15:09:46 -0700 Subject: [PATCH 10/11] new aggregator to find feature_bed_file, also fixed typo in Sample download --- caper/caper/aggregator_main.py | 20 +++++++++++++++++++- caper/caper/views.py | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/caper/caper/aggregator_main.py b/caper/caper/aggregator_main.py index 1c0e585f..f4949b29 100644 --- a/caper/caper/aggregator_main.py +++ b/caper/caper/aggregator_main.py @@ -443,6 +443,7 @@ def json_modifications(self): # update each path in run.json by finding them in outputs folder # separately for feature bed file because location is different feat_basename = os.path.basename(sample_dct['Feature BED file']) + cfiles = os.listdir(self.sample_to_ac_location_dct[sample]) cbf_hits = [x for x in cfiles if x.endswith("_classification_bed_files") and not x.startswith("._")] if cbf_hits: @@ -459,7 +460,11 @@ def json_modifications(self): sample_dct['Feature BED file'] = "Not Provided" else: - sample_dct['Feature BED file'] = "Not Provided" + fp_finding = find_file_by_basename(self.ROOT_FP, feat_basename) + if os.path.exists(fp_finding): + sample_dct['Feature BED file'] = fp_finding + else: + sample_dct['Feature BED file'] = "Not Provided" features_of_interest = [ 'CNV BED file', @@ -468,6 +473,7 @@ def json_modifications(self): 'AA summary file', 'Run metadata JSON', 'Sample metadata JSON', + 'Feature BED file', ] for feature in features_of_interest: @@ -557,6 +563,18 @@ def clean_by_suffix(self, suffix, dir): # except FileNotFoundError: # pass +def find_file_by_basename(directory, basename): + # Walk through the directory + print(f'file to find: {basename}') + for root, dirs, files in os.walk(directory, topdown=True): + # Check if any file matches the given basename + for file in files: + if os.path.basename(file) == basename: + + return os.path.join(root, file) + return None + + # TODO: VALIDATE IS NEVER USED! def validate(): """ diff --git a/caper/caper/views.py b/caper/caper/views.py index c25197a5..e0c39899 100644 --- a/caper/caper/views.py +++ b/caper/caper/views.py @@ -972,7 +972,7 @@ def sample_download(request, project_name, sample_name): png_id = feature['AA_PNG_file'] else: png_id = False - if feature['AA_PNG_file'] != 'Not Provided': + if feature['AA_directory'] != 'Not Provided': aa_directory_id = feature['AA_directory'] else: aa_directory_id = False From 54b1d85c627301a721c0e93908722f60e7669d22 Mon Sep 17 00:00:00 2001 From: Edwin Huang <43209173+edwin5588@users.noreply.github.com> Date: Wed, 23 Oct 2024 15:37:59 -0700 Subject: [PATCH 11/11] update using defaultdict to speed up process --- caper/caper/aggregator_main.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/caper/caper/aggregator_main.py b/caper/caper/aggregator_main.py index f4949b29..0ae1d1fe 100644 --- a/caper/caper/aggregator_main.py +++ b/caper/caper/aggregator_main.py @@ -129,6 +129,8 @@ def __init__(self, filelist, output_name, run_classifier, ref, py3_path, output_ self.py3_path = py3_path self.name_remap = read_name_remap(name_remap_file) + self.feature_beds = defaultdict(str) + self.unzip() if self.run_classifier == "Yes": self.run_amp_classifier() @@ -138,6 +140,7 @@ def __init__(self, filelist, output_name, run_classifier, ref, py3_path, output_ self.sample_to_ac_location_dct = self.aggregate_tables() complete = self.json_modifications() self.cleanup() + if complete: self.complete = True @@ -231,6 +234,9 @@ def locate_dirs_and_metadata_jsons(self): elif f.endswith("_sample_metadata.json"): implied_sname = rchop(f, "_sample_metadata.json") self.samp_mdata_dct[implied_sname] = fp + "/" + f + elif f.endswith("_intervals.bed"): + self.feature_beds[f] = os.path.join(fp, f) + def run_amp_classifier(self): """ @@ -454,15 +460,17 @@ def json_modifications(self): feat_file = feat_file.replace(f'{self.ROOT_FP}/results/', "") sample_dct['Feature BED file'] = feat_file else: + if not feat_file.endswith("/NA"): print(f'Feature: "Feature BED file" {feat_file} doesnt exist for sample {sample_dct["Sample name"]}') sample_dct['Feature BED file'] = "Not Provided" + else: - fp_finding = find_file_by_basename(self.ROOT_FP, feat_basename) - if os.path.exists(fp_finding): - sample_dct['Feature BED file'] = fp_finding + fp_finding = self.feature_beds[feat_basename] + if fp_finding and os.path.exists(fp_finding): + sample_dct['Feature BED file'] = fp_finding else: sample_dct['Feature BED file'] = "Not Provided" @@ -570,7 +578,6 @@ def find_file_by_basename(directory, basename): # Check if any file matches the given basename for file in files: if os.path.basename(file) == basename: - return os.path.join(root, file) return None