URA-872 - initial boilerplate

.coveragerc

@@ -0,0 +1,3 @@
+[run]
+omit = tests/*
+

.github/workflows/pytest.yml

@@ -0,0 +1,21 @@
+name: pytest
+on: [push, pull_request]
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up Python 3.8
+ uses: actions/setup-python@v1
+ with:
+ python-version: 3.8
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install pipenv codecov
+ pip install -r requirements.txt
+ pipenv install --dev
+ - name: Test with pytest
+ run: |
+ pytest -v --cov --count 10 --random-order .

requirements.txt

@@ -0,0 +1,9 @@
+boto3==1.35.27
+pytest==7.0.1
+pytest-cov==4.0.0
+pytest-html==4.1.0
+pytest-metadata==3.0.0
+pytest-mock==3.11.1
+pytest-random-order==1.1.1
+pytest-repeat==0.9.3
+pytest-subtests==0.11.0

s3_upload/s3_upload.py

@@ -0,0 +1,26 @@
+import argparse
+
+
+def parse_args() -> argparse.Namespace:
+ """
+ Parse cmd line arguments
+
+ Returns
+ -------
+ argparse.Namespace
+ parsed arguments
+ """
+ parser = argparse.ArgumentParser()
+
+ # TODO - add the args
+ # need to decide on what running modes to have and user config
+
+ return parser.parse_args()
+
+
+def main() -> None:
+ args = parse_args()
+
+
+if __name__ == "__main__":
+ main()

s3_upload/utils/upload.py

@@ -0,0 +1,71 @@
+"""Functions for handling uploading into S3"""
+
+from concurrent.futures import (
+ ProcessPoolExecutor,
+ ThreadPoolExecutor,
+ wait,
+ as_completed,
+)
+
+import boto3
+
+
+def authenticate():
+ """
+ Authenticate with AWS S3 with given credentials
+ """
+ pass
+
+
+def upload_single_file(local_file):
+ """
+ Uploads single file into S3 storage bucket
+
+ Parameters
+ ----------
+ local_file : _type_
+ _description_
+ """
+ pass
+
+
+def single_core_threaded_upload(files, threads) -> list:
+ """
+ Uploads the given set of `files` to S3 on a single CPU core using
+ maximum of n threads
+
+ Parameters
+ ----------
+ files : list
+ list of local files to upload
+ threads : int
+ maximum number of threaded process to open per core
+
+ Returns
+ -------
+ list
+ _description_
+ """
+ pass
+
+
+def call_by_core(files, cores, threads) -> list:
+ """
+ Call the single_core_threaded_upload on `files` split across n
+ logical CPU cores
+
+ Parameters
+ ----------
+ files : list
+ list of local files to upload
+ cores : int
+ maximum number of logical CPU cores to split uploading across
+ threads : int
+ maximum number of threaded process to open per core
+
+ Returns
+ -------
+ list
+ _description_
+ """
+ pass

s3_upload/utils/utils.py

@@ -0,0 +1,71 @@
+"""General utility functions"""
+
+
+def check_termination_files_exists(dir) -> bool:
+ """
+ _summary_
+
+ Parameters
+ ----------
+ dir : _type_
+ _description_
+
+ Returns
+ -------
+ bool
+ _description_
+ """
+ pass
+
+
+def check_is_sequencing_dir(dir) -> bool:
+ """
+ _summary_
+
+ Parameters
+ ----------
+ dir : _type_
+ _description_
+
+ Returns
+ -------
+ bool
+ _description_
+ """
+ pass
+
+
+def get_sequencing_file_list(dir, exclude_patterns) -> list:
+ """
+ Recursively get list of files and their paths from the given directory
+
+ Parameters
+ ----------
+ dir : _type_
+ _description_
+ exclude_patterns : _type_
+ _description_
+
+ Returns
+ -------
+ list
+ _description_
+ """
+ pass
+
+
+def check_upload_state(dir) -> str:
+ """
+ Checking upload state of run (i.e. complete, partial, not started)
+
+ Parameters
+ ----------
+ dir : _type_
+ _description_
+
+ Returns
+ -------
+ str
+ _description_
+ """
+ pass

tests/__init__.py

@@ -0,0 +1,8 @@
+import sys
+import os
+
+sys.path.append(
+ os.path.abspath(os.path.join(os.path.realpath(__file__), "../../"))
+)
+
+TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "test_data")