From 88c577aa368b57de8fdd0ae1a4e7637bad99381c Mon Sep 17 00:00:00 2001 From: Dylan Dellett-Wion Date: Wed, 30 Oct 2024 09:30:33 -0400 Subject: [PATCH 01/19] Adds pefile to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 7b680b0..cc9dd1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ numpy +pefile flask>=2.0 gunicorn pytest From 366cb7a740a6caa50922e96d5de89b7dd7abea0b Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Fri, 15 Nov 2024 13:39:20 -0500 Subject: [PATCH 02/19] Initial addition of executable editing utils --- app/taskstart.py | 8 ++- app/utils.py | 143 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 141 insertions(+), 10 deletions(-) diff --git a/app/taskstart.py b/app/taskstart.py index a98cd92..5fb22f5 100644 --- a/app/taskstart.py +++ b/app/taskstart.py @@ -12,7 +12,7 @@ from .io import write_metadata, initialize_taskdata from .config import CFG from .routing import routing -from .utils import make_download +from .utils import edit_exe_worker_id, edit_app_worker_id from hashlib import blake2b @@ -29,8 +29,10 @@ def taskstart(): supreme_subid = current_app.config['SUPREME_serializer'].dumps(h_workerId) win_dlpath = os.path.join(CFG['download'], 'win_' + str(session['subId']) + '.exe') mac_dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.app') - make_download(supreme_subid, win_dlpath, 'windows') - make_download(supreme_subid, mac_dlpath, 'mac') + # make_download(supreme_subid, win_dlpath, 'windows') + edit_exe_worker_id(exe_file_path=BASE_EXE_PATH, new_worker_id=supreme_subid, output_file_path=win_dlpath) + # make_download(supreme_subid, mac_dlpath, 'mac') + edit_app_worker_id(app_path=BASE_APP_PATH, new_worker_id=supreme_subid, output_app_path=mac_dlpath) session['dlready'] = True write_metadata(session, ['dlready'], 'a') initialize_taskdata(session) diff --git a/app/utils.py b/app/utils.py index b7d44fb..ca6772a 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,22 +1,26 @@ import random import string import copy +import logging +import os +import plistlib +import shutil +from typing import Optional +from pathlib import Path +from pefile import PE, DIRECTORY_ENTRY + def gen_code(N): """Generate random completion code.""" return ''.join(random.choices(string.ascii_lowercase + string.digits, k=N)) -def make_download(sid, dlpath, platform): - pass - - def pseudorandomize(inblocks, nreps, shuffle_blocks=True, nested_output=False): """ pseudorandomize the input blocks so that each task occurs once before any task is repeated and no tasks occur back to back. Will trigger an infinite loop if the number of blocks it too small to generate enough unique - orders to satisfy the shuffle blocks condition for the number of repititions + orders to satisfy the shuffle blocks condition for the number of repetitions requested. Parameters @@ -24,9 +28,9 @@ def pseudorandomize(inblocks, nreps, shuffle_blocks=True, nested_output=False): inblocks : list of strings or list of list of strings list of tasks to randomize nreps : int - number of repititions of each task + number of repetitions of each task shuffle blocks : bool - Should task order be shuffeled everytime they're repeated + Should task order be shuffled every time they're repeated nested_output : bool Should the output be nested (list of lists) @@ -54,3 +58,128 @@ def pseudorandomize(inblocks, nreps, shuffle_blocks=True, nested_output=False): for task in task_block: flat_blocks.append(task) return flat_blocks + + +logging.basicConfig(level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s') + +def edit_app_worker_id(app_path: str, new_worker_id: str, output_app_path: str) -> None: + """ + Modifies the 'WorkerID' field in the Info.plist of a macOS .app bundle. + + Args: + app_path (str): Path to the original .app bundle whose 'WorkerID' will be modified. + new_worker_id (str): New 'WorkerID' value to replace the existing one. + output_app_path (Optional[str]): Path to save the modified .app bundle. If not provided, + the original bundle will be overwritten. + + Raises: + FileNotFoundError: If the specified .app bundle or Info.plist file does not exist. + ValueError: If the Info.plist file cannot be loaded or parsed. + """ + + # Construct the path to the Info.plist file inside the original .app bundle + plist_path = Path(app_path) / 'Contents' / 'Info.plist' + + # Ensure the .app bundle and Info.plist exist + if not plist_path.exists(): + raise FileNotFoundError(f"The file {plist_path} does not exist.") + + try: + # If an output path is specified, copy the original .app bundle to the new location + output_path = Path(output_app_path) + if output_path.exists(): + raise FileExistsError( + f"The output path {output_app_path} already exists.") + shutil.copytree(app_path, output_app_path) + # Update plist path to the new location + plist_path = output_path / 'Contents' / 'Info.plist' + + + # Load the plist file + with plist_path.open('rb') as plist_file: + plist_data = plistlib.load(plist_file) + + # Log current WorkerID, if present + current_worker_id = plist_data.get('WorkerID', None) + if current_worker_id: + logging.info(f"Current WorkerID: {current_worker_id}") + + # Update the WorkerID + plist_data['WorkerID'] = new_worker_id + + # Write the updated plist back + with plist_path.open('wb') as plist_file: + plistlib.dump(plist_data, plist_file) + + logging.info(f"Successfully updated WorkerID to {new_worker_id}") + + except Exception as e: + raise ValueError(f"Failed to modify the plist file: {e}") + + +def edit_exe_worker_id(exe_file_path: str, new_worker_id: str, output_file_path: str) -> None: + """ + Modifies the 'WorkerID' field in the version information of an executable. + + Args: + exe_file_path (str): Path to the executable whose 'WorkerID' will be modified. + new_worker_id (str): New 'WorkerID' value to replace the existing one. + **Must** be less than or equal to the length of the current 'WorkerID' for a successful update. + output_file_path (str): **Optional**. Path where the modified executable will be saved. + If not provided, the original executable will be overwritten. + + Raises: + FileNotFoundError: If the specified executable does not exist. + ValueError: If the PE file cannot be loaded or parsed. + """ + + if not os.path.exists(exe_file_path): + raise FileNotFoundError(f"The file {exe_file_path} does not exist.") + + output_file_path = Path(output_file_path) + + try: + pe: PE = PE(exe_file_path, fast_load=True) + pe.parse_data_directories( + directories=[DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_RESOURCE']]) + except Exception as e: + raise ValueError(f"Failed to load or parse the PE file: {e}") + + # Access the WorkerID and update it if found + if hasattr(pe, 'FileInfo'): + for file_info in pe.FileInfo: + for info in file_info: + if info.name == 'StringFileInfo': + version_info_dict: dict = info.StringTable[0].entries + worker_id_bytes: bytes = version_info_dict.get( + b'WorkerID', None) + + if worker_id_bytes: + logging.info( + f"Found WorkerID: {worker_id_bytes.decode('utf-8')}") + + # Calculate the original size in bytes and the new value size + original_size: int = len(worker_id_bytes) + new_worker_id_bytes: bytes = new_worker_id.encode( + 'utf-8') + new_size: int = len(new_worker_id_bytes) + + if new_size <= original_size: + # Create the new value padded with null bytes up to the original size + padded_new_worker_id: bytes = new_worker_id_bytes + \ + b'\x00' * (original_size - new_size) + + # Update the value with the padded version + version_info_dict[b"WorkerID"] = padded_new_worker_id + + # Write the updated attribute to the output file + pe.write(output_file_path) + logging.info( + f"Successfully updated WorkerID to {new_worker_id}") + else: + logging.error( + f"Error: New value '{new_worker_id}' is larger than the existing space of {original_size} bytes.") + return + + logging.error(f"Error: WorkerID not found in {exe_file_path}") \ No newline at end of file From d1e7d00e461e67d99fbf014af718eb4bb5f44cc1 Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Fri, 15 Nov 2024 13:43:44 -0500 Subject: [PATCH 03/19] Adds base exe to config & SUPREME.exe to base_executables Co-authored-by: Dylan Nielson Co-authored-by: Dylan Dellett-Wion --- app/config.py | 8 +++++--- app/taskstart.py | 6 ++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/config.py b/app/config.py index 308d158..f40062d 100644 --- a/app/config.py +++ b/app/config.py @@ -29,8 +29,8 @@ if not os.path.isdir(task_badupload_dir): os.makedirs(task_badupload_dir) dl_dir = os.path.join(data_dir, 'download') if not os.path.isdir(dl_dir): os.makedirs(dl_dir) -exe_dir = os.path.join(data_dir, 'exe') -if not os.path.isdir(exe_dir): os.makedirs(exe_dir) +base_exe_dir = os.path.join(data_dir, 'base_executables') +if not os.path.isdir(base_exe_dir): os.makedirs(base_exe_dir) survey_dir = os.path.join(data_dir, 'survey') if not os.path.isdir(survey_dir): os.makedirs(survey_dir) survey_incomplete_dir = os.path.join(survey_dir, 'incomplete') @@ -65,7 +65,9 @@ s_reject=survey_reject_dir, s_complete=survey_complete_dir, download=dl_dir, - exe=dl_dir, + base_exe=os.path.join(base_exe_dir, 'SUPREME.exe'), + # TODO: Add base app to the base_executables directory + # base_app=os.path.join(base_exe_dir, 'SUPREME.app'), disallowed_agents=json.loads(cfg['FLASK']['DISALLOWED_AGENTS']), allowed_agents=json.loads(cfg['FLASK']['ALLOWED_AGENTS']), blocks=json.loads(cfg['SUPREME']['BLOCKS']), diff --git a/app/taskstart.py b/app/taskstart.py index 5fb22f5..354fa60 100644 --- a/app/taskstart.py +++ b/app/taskstart.py @@ -29,10 +29,8 @@ def taskstart(): supreme_subid = current_app.config['SUPREME_serializer'].dumps(h_workerId) win_dlpath = os.path.join(CFG['download'], 'win_' + str(session['subId']) + '.exe') mac_dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.app') - # make_download(supreme_subid, win_dlpath, 'windows') - edit_exe_worker_id(exe_file_path=BASE_EXE_PATH, new_worker_id=supreme_subid, output_file_path=win_dlpath) - # make_download(supreme_subid, mac_dlpath, 'mac') - edit_app_worker_id(app_path=BASE_APP_PATH, new_worker_id=supreme_subid, output_app_path=mac_dlpath) + edit_exe_worker_id(exe_file_path=CFG['base_exe'], new_worker_id=supreme_subid, output_file_path=win_dlpath) + # edit_app_worker_id(app_path=CFG['base_app'], new_worker_id=supreme_subid, output_app_path=mac_dlpath) session['dlready'] = True write_metadata(session, ['dlready'], 'a') initialize_taskdata(session) From 0ce400c608df819a203670c89027ace32b9531f5 Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Fri, 15 Nov 2024 16:08:59 -0500 Subject: [PATCH 04/19] Adds SUPREME.app to base_executables & base_app to CFG Co-author: Dylan Nielson Co-author: Dylan Dellett-Wion --- app/config.py | 3 +-- app/taskstart.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/config.py b/app/config.py index f40062d..893322b 100644 --- a/app/config.py +++ b/app/config.py @@ -66,8 +66,7 @@ s_complete=survey_complete_dir, download=dl_dir, base_exe=os.path.join(base_exe_dir, 'SUPREME.exe'), - # TODO: Add base app to the base_executables directory - # base_app=os.path.join(base_exe_dir, 'SUPREME.app'), + base_app=os.path.join(base_exe_dir, 'SUPREME.dmg'), disallowed_agents=json.loads(cfg['FLASK']['DISALLOWED_AGENTS']), allowed_agents=json.loads(cfg['FLASK']['ALLOWED_AGENTS']), blocks=json.loads(cfg['SUPREME']['BLOCKS']), diff --git a/app/taskstart.py b/app/taskstart.py index 354fa60..5bdcd97 100644 --- a/app/taskstart.py +++ b/app/taskstart.py @@ -30,7 +30,7 @@ def taskstart(): win_dlpath = os.path.join(CFG['download'], 'win_' + str(session['subId']) + '.exe') mac_dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.app') edit_exe_worker_id(exe_file_path=CFG['base_exe'], new_worker_id=supreme_subid, output_file_path=win_dlpath) - # edit_app_worker_id(app_path=CFG['base_app'], new_worker_id=supreme_subid, output_app_path=mac_dlpath) + edit_app_worker_id(app_path=CFG['base_app'], new_worker_id=supreme_subid, output_app_path=mac_dlpath) session['dlready'] = True write_metadata(session, ['dlready'], 'a') initialize_taskdata(session) From 5443ec42837beab52425d5b2c97bb25eb5f1131c Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Mon, 18 Nov 2024 12:34:24 -0500 Subject: [PATCH 05/19] add dmg tools to dockerfile --- Dockerfile | 7 ++++++- app/config.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2b8c84b..2f2da5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,12 @@ WORKDIR /cogmood_backend COPY requirements.txt . RUN pip install -r requirements.txt &&\ playwright install && \ - playwright install-deps + playwright install-deps && \ + apt-get install gcc g++ cmake zlib1g-dev genisoimage && \ + git clone https://github.com/hamstergene/libdmg-hfsplus.git && \ + cd libdmg-hfsplus && \ + cmake . && \ + make COPY . . ENV SCRIPT_NAME="/cogmood" CMD ["gunicorn", "-b", "0.0.0.0:8000", "-w", "10", "-k", "gevent", "--max-requests", "5000", "app:app"] diff --git a/app/config.py b/app/config.py index 893322b..aca7b3f 100644 --- a/app/config.py +++ b/app/config.py @@ -66,7 +66,7 @@ s_complete=survey_complete_dir, download=dl_dir, base_exe=os.path.join(base_exe_dir, 'SUPREME.exe'), - base_app=os.path.join(base_exe_dir, 'SUPREME.dmg'), + base_app=os.path.join(base_exe_dir, 'SUPREME.app'), disallowed_agents=json.loads(cfg['FLASK']['DISALLOWED_AGENTS']), allowed_agents=json.loads(cfg['FLASK']['ALLOWED_AGENTS']), blocks=json.loads(cfg['SUPREME']['BLOCKS']), From e83d1da5b87f98b62dd2c0d9fc4580e7f5cb551d Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 18 Nov 2024 13:53:18 -0500 Subject: [PATCH 06/19] add dmg tools to dockerfile --- Dockerfile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2f2da5a..dcf8002 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,12 +4,17 @@ WORKDIR /cogmood_backend COPY requirements.txt . RUN pip install -r requirements.txt &&\ playwright install && \ - playwright install-deps && \ - apt-get install gcc g++ cmake zlib1g-dev genisoimage && \ - git clone https://github.com/hamstergene/libdmg-hfsplus.git && \ + playwright install-deps +RUN apt-get install -y cmake genisoimage && \ + cd / && \ + git clone https://github.com/planetbeing/libdmg-hfsplus.git && \ cd libdmg-hfsplus && \ + git fetch && \ + git checkout openssl-1.1 && \ cmake . && \ - make + make && \ + mv dmg/dmg /usr/local/bin/ && \ + cd ../cogmood_backend COPY . . ENV SCRIPT_NAME="/cogmood" CMD ["gunicorn", "-b", "0.0.0.0:8000", "-w", "10", "-k", "gevent", "--max-requests", "5000", "app:app"] From 373a7bd0948f66d500ba9631332e79af1639292a Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Mon, 18 Nov 2024 14:08:47 -0500 Subject: [PATCH 07/19] build dmg for modified app --- app/taskstart.py | 4 +-- app/utils.py | 84 +++++++++++++++++++++++++++++++----------------- 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/app/taskstart.py b/app/taskstart.py index 5bdcd97..fded26f 100644 --- a/app/taskstart.py +++ b/app/taskstart.py @@ -28,7 +28,7 @@ def taskstart(): h_workerId = blake2b(session['workerId'].encode(), digest_size=24).hexdigest() supreme_subid = current_app.config['SUPREME_serializer'].dumps(h_workerId) win_dlpath = os.path.join(CFG['download'], 'win_' + str(session['subId']) + '.exe') - mac_dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.app') + mac_dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.dmg') edit_exe_worker_id(exe_file_path=CFG['base_exe'], new_worker_id=supreme_subid, output_file_path=win_dlpath) edit_app_worker_id(app_path=CFG['base_app'], new_worker_id=supreme_subid, output_app_path=mac_dlpath) session['dlready'] = True @@ -48,7 +48,7 @@ def taskstart(): @bp.route('/download/mac') def download_mac(): - dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.app') + dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.dmg') session['dlstarted'] = True write_metadata(session, ['dlstarted'], 'a') diff --git a/app/utils.py b/app/utils.py index ca6772a..9dd47f3 100644 --- a/app/utils.py +++ b/app/utils.py @@ -5,6 +5,8 @@ import os import plistlib import shutil +from subprocess import run +from tempfile import TemporaryDirectory from typing import Optional from pathlib import Path from pefile import PE, DIRECTORY_ENTRY @@ -63,15 +65,14 @@ def pseudorandomize(inblocks, nreps, shuffle_blocks=True, nested_output=False): logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') -def edit_app_worker_id(app_path: str, new_worker_id: str, output_app_path: str) -> None: +def edit_app_worker_id(app_path: str, new_worker_id: str, output_dmg_path: str) -> None: """ Modifies the 'WorkerID' field in the Info.plist of a macOS .app bundle. Args: app_path (str): Path to the original .app bundle whose 'WorkerID' will be modified. new_worker_id (str): New 'WorkerID' value to replace the existing one. - output_app_path (Optional[str]): Path to save the modified .app bundle. If not provided, - the original bundle will be overwritten. + output_app_path (Optional[str]): Path to save the modified .app bundle in a dmg. Raises: FileNotFoundError: If the specified .app bundle or Info.plist file does not exist. @@ -86,33 +87,56 @@ def edit_app_worker_id(app_path: str, new_worker_id: str, output_app_path: str) raise FileNotFoundError(f"The file {plist_path} does not exist.") try: - # If an output path is specified, copy the original .app bundle to the new location - output_path = Path(output_app_path) - if output_path.exists(): - raise FileExistsError( - f"The output path {output_app_path} already exists.") - shutil.copytree(app_path, output_app_path) - # Update plist path to the new location - plist_path = output_path / 'Contents' / 'Info.plist' - - - # Load the plist file - with plist_path.open('rb') as plist_file: - plist_data = plistlib.load(plist_file) - - # Log current WorkerID, if present - current_worker_id = plist_data.get('WorkerID', None) - if current_worker_id: - logging.info(f"Current WorkerID: {current_worker_id}") - - # Update the WorkerID - plist_data['WorkerID'] = new_worker_id - - # Write the updated plist back - with plist_path.open('wb') as plist_file: - plistlib.dump(plist_data, plist_file) - - logging.info(f"Successfully updated WorkerID to {new_worker_id}") + # copy the app to a directory within a temp directory to create the dmg + with TemporaryDirectory() as tmpdir: + imgdir = Path(tmpdir) / 'SUPREME' + imgdir.mkdir() + img_path = tmpdir / 'SUPREME.img' + img_app_path = imgdir / 'SUPREME.app' + shutil.copytree(app_path, img_app_path) + + plist_path = img_app_path / 'Contents' / 'Info.plist' + + # Load the plist file + with plist_path.open('rb') as plist_file: + plist_data = plistlib.load(plist_file) + + # Log current WorkerID, if present + current_worker_id = plist_data.get('WorkerID', None) + if current_worker_id: + logging.info(f"Current WorkerID: {current_worker_id}") + + # Update the WorkerID + plist_data['WorkerID'] = new_worker_id + + # Write the updated plist back + with plist_path.open('wb') as plist_file: + plistlib.dump(plist_data, plist_file) + + logging.info(f"Successfully updated WorkerID to {new_worker_id}") + + iso_cmd = [ + 'genisoimage', + '-D', + '-V', + '"SUPREME"', + '-no-pad', + '-r', + '-apple', + '-file-mode', + '0777', + '-o', + img_path, + imgdir + ] + run(iso_cmd, check=True) + + dmg_cmd = [ + 'dmg', + 'dmg', + img_path, + output_dmg_path + ] except Exception as e: raise ValueError(f"Failed to modify the plist file: {e}") From 3bbf568ef208658c9c0f69ee10b2a020da7dc5bd Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 18 Nov 2024 14:25:09 -0500 Subject: [PATCH 08/19] fix mac dl --- app/taskstart.py | 2 +- app/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/taskstart.py b/app/taskstart.py index fded26f..74bab88 100644 --- a/app/taskstart.py +++ b/app/taskstart.py @@ -30,7 +30,7 @@ def taskstart(): win_dlpath = os.path.join(CFG['download'], 'win_' + str(session['subId']) + '.exe') mac_dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.dmg') edit_exe_worker_id(exe_file_path=CFG['base_exe'], new_worker_id=supreme_subid, output_file_path=win_dlpath) - edit_app_worker_id(app_path=CFG['base_app'], new_worker_id=supreme_subid, output_app_path=mac_dlpath) + edit_app_worker_id(app_path=CFG['base_app'], new_worker_id=supreme_subid, output_dmg_path=mac_dlpath) session['dlready'] = True write_metadata(session, ['dlready'], 'a') initialize_taskdata(session) diff --git a/app/utils.py b/app/utils.py index 9dd47f3..6d99e1f 100644 --- a/app/utils.py +++ b/app/utils.py @@ -91,7 +91,7 @@ def edit_app_worker_id(app_path: str, new_worker_id: str, output_dmg_path: str) with TemporaryDirectory() as tmpdir: imgdir = Path(tmpdir) / 'SUPREME' imgdir.mkdir() - img_path = tmpdir / 'SUPREME.img' + img_path = Path(tmpdir) / 'SUPREME.img' img_app_path = imgdir / 'SUPREME.app' shutil.copytree(app_path, img_app_path) @@ -206,4 +206,4 @@ def edit_exe_worker_id(exe_file_path: str, new_worker_id: str, output_file_path: f"Error: New value '{new_worker_id}' is larger than the existing space of {original_size} bytes.") return - logging.error(f"Error: WorkerID not found in {exe_file_path}") \ No newline at end of file + logging.error(f"Error: WorkerID not found in {exe_file_path}") From 259b0d798337061ee0118c70ae5347cd76d138d0 Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Mon, 18 Nov 2024 14:56:34 -0500 Subject: [PATCH 09/19] run dmg --- app/taskstart.py | 8 ++++---- app/utils.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/taskstart.py b/app/taskstart.py index 74bab88..3fe2e5d 100644 --- a/app/taskstart.py +++ b/app/taskstart.py @@ -35,8 +35,8 @@ def taskstart(): write_metadata(session, ['dlready'], 'a') initialize_taskdata(session) - mac_link = url_for('taskstart.download_mac') - win_link = url_for('taskstart.download_win') + mac_link = url_for('taskstart.download_mac', **request.args) + win_link = url_for('taskstart.download_win', **request.args) if rres is None: return render_template('taskstart.html', @@ -60,7 +60,7 @@ def download_mac(): mimetype="inode/directory" ) else: - return redirect(url_for('task.task', **request.args)) + return redirect(url_for('taskstart.taskstart', **request.args)) @bp.route('/download/win') def download_win(): @@ -76,4 +76,4 @@ def download_win(): mimetype="application/vnd.microsoft.portable-executable" ) else: - return redirect(url_for('task.task', **request.args)) + return redirect(url_for('taskstart.taskstart', **request.args)) diff --git a/app/utils.py b/app/utils.py index 6d99e1f..40a22c0 100644 --- a/app/utils.py +++ b/app/utils.py @@ -137,6 +137,7 @@ def edit_app_worker_id(app_path: str, new_worker_id: str, output_dmg_path: str) img_path, output_dmg_path ] + run(dmg_cmd, check=True) except Exception as e: raise ValueError(f"Failed to modify the plist file: {e}") From e4ab628eca7aeb92cc398627418b3c3fa190766e Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Mon, 18 Nov 2024 15:10:58 -0500 Subject: [PATCH 10/19] fix download name/mimetype --- app/taskstart.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/taskstart.py b/app/taskstart.py index 3fe2e5d..d2f9a9d 100644 --- a/app/taskstart.py +++ b/app/taskstart.py @@ -56,8 +56,8 @@ def download_mac(): return send_file( dlpath, as_attachment=True, - download_name='CogMood_task', - mimetype="inode/directory" + download_name='CogMood_task.dmg', + mimetype="application/octet-stream" ) else: return redirect(url_for('taskstart.taskstart', **request.args)) From e3e5231f0a3db123ca8b9654c1093e5ada071015 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 19 Nov 2024 10:38:11 -0500 Subject: [PATCH 11/19] change download name to SUPREME --- app/taskstart.py | 2 +- app/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/taskstart.py b/app/taskstart.py index d2f9a9d..345b130 100644 --- a/app/taskstart.py +++ b/app/taskstart.py @@ -56,7 +56,7 @@ def download_mac(): return send_file( dlpath, as_attachment=True, - download_name='CogMood_task.dmg', + download_name='SUPREME.dmg', mimetype="application/octet-stream" ) else: diff --git a/app/utils.py b/app/utils.py index 40a22c0..a9407b7 100644 --- a/app/utils.py +++ b/app/utils.py @@ -119,7 +119,7 @@ def edit_app_worker_id(app_path: str, new_worker_id: str, output_dmg_path: str) 'genisoimage', '-D', '-V', - '"SUPREME"', + 'SUPREME', '-no-pad', '-r', '-apple', From a516d7698b81c6077376ad9f58ba4e5484988633 Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Fri, 13 Dec 2024 10:23:21 -0500 Subject: [PATCH 12/19] extend timeouts --- app/templates/taskstart.html | 2 ++ app/tests/test_app.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/templates/taskstart.html b/app/templates/taskstart.html index 538611e..94627e5 100644 --- a/app/templates/taskstart.html +++ b/app/templates/taskstart.html @@ -93,6 +93,8 @@

Windows Instructions:

+ +

macOS Instructions:

diff --git a/app/tests/test_app.py b/app/tests/test_app.py index 034cbcf..b04387b 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -100,7 +100,7 @@ def test_survey_complete(url, server, page: Page, request): page.get_by_role("button", name="Complete").click() else: page.get_by_role("button", name="Next").click() - expect(page).to_have_url(f'{url}taskstart?PROLIFIC_PID={workerId}') + expect(page).to_have_url(f'{url}taskstart?PROLIFIC_PID={workerId}', timeout=10000) if server: # get saved data and compare to expectation @@ -254,7 +254,7 @@ def test_taskcontrol(url, server, loadtest, ignore_https_errors, page: Page, req page.get_by_role("button", name="Complete").click() else: page.get_by_role("button", name="Next").click() - expect(page).to_have_url(f'{url}taskstart?PROLIFIC_PID={workerId}') + expect(page).to_have_url(f'{url}taskstart?PROLIFIC_PID={workerId}', timeout=10000) # mock task communication with server cfg = configparser.ConfigParser() From 50a5282ed5e712fbd94da431bc46c766852a979d Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Fri, 13 Dec 2024 11:58:59 -0500 Subject: [PATCH 13/19] update validation rules --- app/validation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/validation.py b/app/validation.py index 77dd068..7dc4a10 100644 --- a/app/validation.py +++ b/app/validation.py @@ -7,7 +7,7 @@ FLKR_KEYS = {'run_num', 'appear_time_time', 'appear_time_error', 'disappear_time', 'pressed', 'press_time_time','press_time_error', 'rt', 'block', 'trial_id', 'correct', 'fmri_tr_time', 'eeg_pulse_time', 'log_time','loc_x', 'loc_y', 'condition', - 'stim', 'dir', 'corr_resp', 'log_num'} + 'dir', 'corr_resp', 'log_num'} RDM_THR = 13 RDM_NTRIALS = 68 RDM_KEYS = {'run_num', 'appear_time_time', 'appear_time_error', 'disappear_time_time', @@ -35,7 +35,7 @@ def validate_flkr(zipped_path): with tempfile.TemporaryDirectory() as tmpdir: slog_file = ZipFile(zipped_path).extract('log_flkr_0.slog', path=tmpdir) lod = log.log2dl(slog_file) - if len(lod) != FLKR_NTRIALS: + if len(lod) <= FLKR_NTRIALS: return False, 'ntrials' if lod[0].keys() != FLKR_KEYS: return False, 'keys' @@ -49,7 +49,7 @@ def validate_rdm(zipped_path): with tempfile.TemporaryDirectory() as tmpdir: slog_file = ZipFile(zipped_path).extract('log_rdm_0.slog', path=tmpdir) lod = log.log2dl(slog_file) - if len(lod) != RDM_NTRIALS: + if len(lod) <= RDM_NTRIALS: return False, 'ntrials' if lod[0].keys() != RDM_KEYS: return False, 'keys' @@ -66,7 +66,7 @@ def validate_rdm(zipped_path): def validate_bart(zipped_path): with tempfile.TemporaryDirectory() as tmpdir: slog_file = ZipFile(zipped_path).extract('log_bart_0.slog', path=tmpdir) - lod =log.log2dl(slog_file) + lod = log.log2dl(slog_file) if lod[0].keys() != BART_KEYS: return False, 'keys' pressed_j = False @@ -90,7 +90,7 @@ def validate_cab(zipped_path): with tempfile.TemporaryDirectory() as tmpdir: slog_file = ZipFile(zipped_path).extract('log_cab_0.slog', path=tmpdir) lod = log.log2dl(slog_file) - if len(lod) != CAB_NTRIALS: + if len(lod) <= CAB_NTRIALS: return False, 'ntrials' if lod[0].keys() != CAB_KEYS: return False, 'keys' From fe74f0352e322b240e34a0dd3f13b8560dec5fce Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Fri, 13 Dec 2024 12:17:16 -0500 Subject: [PATCH 14/19] fix taskstart routing --- app/taskstart.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/app/taskstart.py b/app/taskstart.py index 345b130..923247c 100644 --- a/app/taskstart.py +++ b/app/taskstart.py @@ -25,20 +25,21 @@ def taskstart(): """Present download to participant.""" rres = routing('taskstart') - h_workerId = blake2b(session['workerId'].encode(), digest_size=24).hexdigest() - supreme_subid = current_app.config['SUPREME_serializer'].dumps(h_workerId) - win_dlpath = os.path.join(CFG['download'], 'win_' + str(session['subId']) + '.exe') - mac_dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.dmg') - edit_exe_worker_id(exe_file_path=CFG['base_exe'], new_worker_id=supreme_subid, output_file_path=win_dlpath) - edit_app_worker_id(app_path=CFG['base_app'], new_worker_id=supreme_subid, output_dmg_path=mac_dlpath) - session['dlready'] = True - write_metadata(session, ['dlready'], 'a') - initialize_taskdata(session) + if rres is None: + h_workerId = blake2b(session['workerId'].encode(), digest_size=24).hexdigest() + supreme_subid = current_app.config['SUPREME_serializer'].dumps(h_workerId) + if not session['dlready']: + win_dlpath = os.path.join(CFG['download'], 'win_' + str(session['subId']) + '.exe') + mac_dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.dmg') + edit_exe_worker_id(exe_file_path=CFG['base_exe'], new_worker_id=supreme_subid, output_file_path=win_dlpath) + edit_app_worker_id(app_path=CFG['base_app'], new_worker_id=supreme_subid, output_dmg_path=mac_dlpath) + session['dlready'] = True + write_metadata(session, ['dlready'], 'a') + initialize_taskdata(session) - mac_link = url_for('taskstart.download_mac', **request.args) - win_link = url_for('taskstart.download_win', **request.args) + mac_link = url_for('taskstart.download_mac', **request.args) + win_link = url_for('taskstart.download_win', **request.args) - if rres is None: return render_template('taskstart.html', platform=session['platform'], mac_link=mac_link, From bbe254454bf32116635217be7fd2432e9fb553fb Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Fri, 13 Dec 2024 12:20:45 -0500 Subject: [PATCH 15/19] fix taskstart routing --- app/taskstart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/taskstart.py b/app/taskstart.py index 923247c..6c06a61 100644 --- a/app/taskstart.py +++ b/app/taskstart.py @@ -28,7 +28,7 @@ def taskstart(): if rres is None: h_workerId = blake2b(session['workerId'].encode(), digest_size=24).hexdigest() supreme_subid = current_app.config['SUPREME_serializer'].dumps(h_workerId) - if not session['dlready']: + if 'dlready' not in session or not session['dlready']: win_dlpath = os.path.join(CFG['download'], 'win_' + str(session['subId']) + '.exe') mac_dlpath = os.path.join(CFG['download'], 'mac_' + str(session['subId']) + '.dmg') edit_exe_worker_id(exe_file_path=CFG['base_exe'], new_worker_id=supreme_subid, output_file_path=win_dlpath) From cc90379735cc8581c1088d2a9037bee3d4ac1d83 Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Fri, 13 Dec 2024 13:37:40 -0500 Subject: [PATCH 16/19] add route debugging --- app/app.ini | 4 ++++ app/complete.py | 8 ++++++++ app/config.py | 1 + app/routing.py | 39 ++++++++++++++++++++++++++++++++++----- 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/app/app.ini b/app/app.ini index 3f917a8..b0f5f1e 100644 --- a/app/app.ini +++ b/app/app.ini @@ -12,6 +12,10 @@ ALLOW_RESTART = true # Accepts true or false DEBUG = true +# Toggle route debug mode (way more logging from routing.py) +# Accepts true or false +ROUTE_DEBUG = false + # allowed user agents [default: ["macintosh", "windows"]] ALLOWED_AGENTS = ["macintosh", "windows"] diff --git a/app/complete.py b/app/complete.py index 8fd5255..c27ab40 100644 --- a/app/complete.py +++ b/app/complete.py @@ -1,6 +1,7 @@ from flask import (Blueprint, redirect, render_template, request, session, url_for) from .io import write_metadata from .config import CFG +from .utils import logging ## Initialize blueprint. bp = Blueprint('complete', __name__) @@ -9,6 +10,13 @@ def complete(): """Present completion screen to participant.""" + route_debug = CFG['route_debug'] + if route_debug: + logging.info(f'Request to complete') + logging.info('Session contents') + for k, v in session.items(): + logging.info(f"{k}: {v}") + # Case 1: Error-catching: screen for missing session. if not 'workerId' in session: diff --git a/app/config.py b/app/config.py index aca7b3f..6cd833f 100644 --- a/app/config.py +++ b/app/config.py @@ -44,6 +44,7 @@ CFG = dict( debug=cfg['FLASK'].getboolean('DEBUG'), + route_debug=cfg['FLASK'].getboolean('ROUTE_DEBUG'), allow_restart=cfg['FLASK'].getboolean('ALLOW_RESTART'), code_success=cfg['PROLIFIC']['CODE_SUCCESS'], code_reject=cfg['PROLIFIC']['CODE_REJECT'], diff --git a/app/routing.py b/app/routing.py index 54c974a..1a61734 100644 --- a/app/routing.py +++ b/app/routing.py @@ -5,10 +5,13 @@ from .io import write_metadata from .config import CFG from .utils import gen_code - +from .utils import logging def routing(ep): """Unify the routing to reduce repetition""" + route_debug = CFG['route_debug'] + if route_debug: + logging.info(f'Routing request for: {ep}') info = dict( workerId = request.args.get('PROLIFIC_PID'), # Prolific metadata assignmentId = request.args.get('SESSION_ID'), # Prolific metadata @@ -17,7 +20,13 @@ def routing(ep): address = request.remote_addr, # NivTurk metadata user_agent = request.user_agent.string.lower(), # User metadata ) - + if route_debug: + logging.info('Info contents') + for k,v in info.items(): + logging.info(f"{k}: {v}") + logging.info('Session contents') + for k,v in session.items(): + logging.info(f"{k}: {v}") disallowed_agent = any([device in info['user_agent'] for device in CFG['disallowed_agents']]) allowed_agent = any([device in info['user_agent'] for device in CFG['allowed_agents']]) @@ -25,14 +34,34 @@ def routing(ep): try: h_workerId = blake2b(info['workerId'].encode(), digest_size=24).hexdigest() except AttributeError: + if route_debug: + logging.info('Failed case 1, no PROLIFIC_PID in url') ## Redirect participant to error (missing workerId). return redirect(url_for('error.error', errornum=1000)) + # complete might be written to metadata without updating session + # because task uploading has no session + if h_workerId in os.listdir(CFG['meta']): + + ## Parse log file. + with open(os.path.join(CFG['meta'], h_workerId), 'r') as f: + logs = f.read() + + # Grab str fields from logs + fields = [ + 'complete', + ] + + for field in fields: + re_res = re.search(f'\t{field}\t(.*)\n', logs) + if re_res: + session[field] = re_res.group(1) # Case 2: mobile / tablet / game console user. # Not a terminal error because we want the user to be able to try again from a different device if disallowed_agent or not allowed_agent: - + if route_debug: + logging.info(f'Failed case 2, agent {info["user_agent"]} not allowed') # Redirect participant to error (platform error). return redirect(url_for('error.error', errornum=1001)) @@ -83,7 +112,7 @@ def routing(ep): ] for field in bool_fields: - re_res = re.search(f'{field}\t(.*)\n', logs) + re_res = re.search(f'\t{field}\t(.*)\n', logs) if re_res and re_res.group(1) == 'True': info[field] = True # consent = true elif re_res and re_res.group(1) == 'False': @@ -99,7 +128,7 @@ def routing(ep): ] for field in fields: - re_res = re.search(f'{field}\t(.*)\n', logs) + re_res = re.search(f'\t{field}\t(.*)\n', logs) if re_res: info[field] = re_res.group(1) From 9aa992aa5597db50dafd382842949a8c30f2703f Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Fri, 13 Dec 2024 13:37:56 -0500 Subject: [PATCH 17/19] click the download button --- app/tests/test_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/test_app.py b/app/tests/test_app.py index b04387b..69681d0 100644 --- a/app/tests/test_app.py +++ b/app/tests/test_app.py @@ -255,7 +255,7 @@ def test_taskcontrol(url, server, loadtest, ignore_https_errors, page: Page, req else: page.get_by_role("button", name="Next").click() expect(page).to_have_url(f'{url}taskstart?PROLIFIC_PID={workerId}', timeout=10000) - + page.get_by_role("button").click() # mock task communication with server cfg = configparser.ConfigParser() cfg.read(os.path.join(ROOT_DIR, 'app.ini')) From ce1ab57929faf4f71a6f2b6a963e0c2d82369831 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 13 Dec 2024 18:04:57 -0500 Subject: [PATCH 18/19] fix mac signing --- Dockerfile | 6 ++++++ app/utils.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/Dockerfile b/Dockerfile index dcf8002..53f7559 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,13 @@ RUN apt-get install -y cmake genisoimage && \ cmake . && \ make && \ mv dmg/dmg /usr/local/bin/ && \ + cd / +RUN curl https://sh.rustup.rs -sSf > install_rust.sh && \ + bash install_rust.sh -y && \ + . "$HOME/.cargo/env" && \ + cargo install --git https://github.com/indygreg/apple-platform-rs --branch main --bin rcodesign apple-codesign && \ cd ../cogmood_backend + COPY . . ENV SCRIPT_NAME="/cogmood" CMD ["gunicorn", "-b", "0.0.0.0:8000", "-w", "10", "-k", "gevent", "--max-requests", "5000", "app:app"] diff --git a/app/utils.py b/app/utils.py index a9407b7..2a5458b 100644 --- a/app/utils.py +++ b/app/utils.py @@ -115,6 +115,13 @@ def edit_app_worker_id(app_path: str, new_worker_id: str, output_dmg_path: str) logging.info(f"Successfully updated WorkerID to {new_worker_id}") + sign_cmd = [ + '/root/.cargo/bin/rcodesign', + 'sign', + img_app_path + ] + run(sign_cmd) + iso_cmd = [ 'genisoimage', '-D', From 3e49f4be1804fbca4b4c297ba3cf5da7d424bf0f Mon Sep 17 00:00:00 2001 From: Dylan Nielson Date: Mon, 16 Dec 2024 14:11:58 -0500 Subject: [PATCH 19/19] don't log IP addresses in metadata --- app/routing.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/routing.py b/app/routing.py index 1a61734..9d34a42 100644 --- a/app/routing.py +++ b/app/routing.py @@ -17,7 +17,6 @@ def routing(ep): assignmentId = request.args.get('SESSION_ID'), # Prolific metadata hitId = request.args.get('STUDY_ID'), # Prolific metadata subId = gen_code(24), # NivTurk metadata - address = request.remote_addr, # NivTurk metadata user_agent = request.user_agent.string.lower(), # User metadata ) if route_debug: @@ -144,7 +143,6 @@ def routing(ep): 'hitId', 'assignmentId', 'subId', - 'address', 'user_agent', 'platform' ], 'w') @@ -165,7 +163,6 @@ def routing(ep): 'hitId', 'assignmentId', 'subId', - 'address', 'user_agent', 'platform' ], 'w')