Enhance input validation, CLI functionality, and testing for directLFQ

.github/workflows/build_windows_package.yml

@@ -18,6 +18,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Master version bumped
  id: master_version_bumped
  shell: bash -l {0}
@@ -52,6 +53,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Conda info
  shell: bash -l {0}
  run: conda info

.github/workflows/nbdev_tests.yml

@@ -19,6 +19,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Conda info
  shell: bash -l {0}
  run: conda info

.github/workflows/publish_and_release.yml

@@ -18,6 +18,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Master version bumped
  id: master_version_bumped
  shell: bash -l {0}
@@ -51,6 +52,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Conda info
  shell: bash -l {0}
  run: conda info
@@ -83,6 +85,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Conda info
  shell: bash -l {0}
  run: conda info
@@ -115,6 +118,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Conda info
  shell: bash -l {0}
  run: conda info
@@ -150,6 +154,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Conda info
  shell: bash -l {0}
  run: conda info
@@ -185,6 +190,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Conda info
  shell: bash -l {0}
  run: conda info

.github/workflows/quick_tests.yml

@@ -20,6 +20,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Conda info
  shell: bash -l {0}
  run: conda info
@@ -45,6 +46,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Conda info
  shell: bash -l {0}
  run: conda info

.github/workflows/quick_tests_ubuntu.yml

@@ -20,6 +20,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Conda info
  shell: bash -l {0}
  run: conda info

.github/workflows/unused/all_tests.yml

@@ -20,6 +20,7 @@ jobs:
  with:
  auto-update-conda: true
  python-version: ${{ matrix.python-version }}
+ miniconda-version: latest
  - name: Conda info
  shell: bash -l {0}
  run: conda info

directlfq/__init__.py

@@ -2,7 +2,7 @@
 
 
 __project__ = "directlfq"
-__version__ = "0.2.19"
+__version__ = "0.2.20"
 __license__ = "Apache"
 __description__ = "An open-source Python package of the AlphaPept ecosystem"
 __author__ = "Mann Labs"

directlfq/cli.py

@@ -197,7 +197,7 @@ def gui():
 @click.option("--filename_suffix", "-fs", type=str, default="", help="A suffix to add to the output file name.")
 @click.option("--num_cores", "-nc", type = int, default = None, help="The number of cores to use (default is to use multiprocessing).")
 @click.option("--deactivate_normalization", "-dn", type = bool, default = False, help="If you want to deactivate the normalization step, you can set this flag to True.")
-@click.option("--filter_dict", "-dn", type = bool, default = False, help="In case you want to define specific filters in addition to the standard filters, you can add a yaml file where the filters are defined (see GitHub docu for example).")
+@click.option("--filter_dict", "-fd", type = str, default = None, help="In case you want to define specific filters in addition to the standard filters, you can add a yaml file where the filters are defined (see GitHub docu for example).")
 
 def run_directlfq(**kwargs):
  print("starting directLFQ")

directlfq/lfq_manager.py

@@ -48,6 +48,7 @@ def run_lfq(input_file, columns_to_add = [], selected_proteins_file :str = None
  input_df = lfqutils.import_data(input_file=input_file, input_type_to_use=input_type_to_use, filter_dict=filter_dict)
 
  input_df = lfqutils.sort_input_df_by_protein_id(input_df)
+ input_df = lfqutils.remove_potential_quant_id_duplicates(input_df)
  input_df = lfqutils.index_and_log_transform_input_df(input_df)
  input_df = lfqutils.remove_allnan_rows_input_df(input_df)
 

directlfq/normalization.py

@@ -259,10 +259,15 @@ def __init__(self, complete_dataframe, num_samples_quadratic = 100):
  self.normalization_function = None
 
  def _run_normalization(self):
+ self._check_that_there_are_no_duplicate_rows()
  if len(self.complete_dataframe.index) <= self._num_samples_quadratic:
  self._normalize_complete_input_quadratic()
  else:
  self._normalize_quadratic_and_linear()
+
+ def _check_that_there_are_no_duplicate_rows(self):
+ if self.complete_dataframe.index.duplicated().any():
+ raise ValueError("There are duplicate rows in the input dataframe. Ensure that there are no duplicate quant_id/ion values.")
 
  def _normalize_complete_input_quadratic(self):
  self.complete_dataframe = self.normalization_function(self.complete_dataframe)
@@ -295,6 +300,8 @@ def _shift_remaining_dataframe_to_reference_sample(self):
  linear_shifted_dataframe = SampleShifterLinear(linear_subset_dataframe, self._merged_reference_sample).ion_dataframe
  self.complete_dataframe.loc[ self._linear_subset_rows, :] = linear_shifted_dataframe
 
+
+
  @staticmethod
  @njit
  def _get_num_nas_in_row(row):

directlfq/utils.py

@@ -310,6 +310,30 @@ def index_and_log_transform_input_df(data_df):
 def remove_allnan_rows_input_df(data_df):
  return data_df.dropna(axis = 0, how = 'all')
 
+def remove_potential_quant_id_duplicates(data_df : pd.DataFrame):
+ """
+ Remove duplicate entries from a DataFrame based on the QUANT_ID column.
+
+ This function removes duplicate rows from the input DataFrame, keeping only the first
+ occurrence of each unique QUANT_ID. It also logs a warning message if any duplicates
+ are found and removed.
+
+ Args:
+ data_df (pd.DataFrame): dataframe in directLFQ format
+
+ Returns:
+ pd.DataFrame: dataframe in directLFQ format w duplicate QUANT_ID entries removed.
+ """
+ before_drop = len(data_df)
+ data_df = data_df.drop_duplicates(subset=config.QUANT_ID, keep='first')
+ after_drop = len(data_df)
+ if before_drop != after_drop:
+ entries_removed = before_drop - after_drop
+ LOGGER.warning(f"Duplicate quant_ids detected. {entries_removed} rows removed from input df.")
+
+ return data_df
+
+
 def sort_input_df_by_protein_id(data_df):
  return data_df.sort_values(by = config.PROTEIN_ID,ignore_index=True)
 

misc/bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.19
+current_version = 0.2.20
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?

release/one_click_linux_gui/control

@@ -1,5 +1,5 @@
 Package: directlfq
-Version: 0.2.19
+Version: 0.2.20
 Architecture: all
 Maintainer: Mann Labs <[email protected]>
 Description: directlfq

release/one_click_linux_gui/create_installer_linux.sh

@@ -17,7 +17,7 @@ python setup.py sdist bdist_wheel
 # Setting up the local package
 cd release/one_click_linux_gui
 # Make sure you include the required extra packages and always use the stable or very-stable options!
-pip install "../../dist/directlfq-0.2.19-py3-none-any.whl[stable, gui]"
+pip install "../../dist/directlfq-0.2.20-py3-none-any.whl[stable, gui]"
 
 # Creating the stand-alone pyinstaller folder
 pip install pyinstaller==4.10

release/one_click_macos_gui/Info.plist

@@ -9,9 +9,9 @@
  <key>CFBundleIconFile</key>
  <string>alpha_logo.icns</string>
  <key>CFBundleIdentifier</key>
- <string>directlfq.0.2.19</string>
+ <string>directlfq.0.2.20</string>
  <key>CFBundleShortVersionString</key>
- <string>0.2.19</string>
+ <string>0.2.20</string>
  <key>CFBundleInfoDictionaryVersion</key>
  <string>6.0</string>
  <key>CFBundleName</key>

release/one_click_macos_gui/create_installer_macos.sh

@@ -20,7 +20,7 @@ python setup.py sdist bdist_wheel
 
 # Setting up the local package
 cd release/one_click_macos_gui
-pip install "../../dist/directlfq-0.2.19-py3-none-any.whl[stable, gui]"
+pip install "../../dist/directlfq-0.2.20-py3-none-any.whl[stable, gui]"
 
 # Creating the stand-alone pyinstaller folder
 pip install pyinstaller==4.10
@@ -40,5 +40,5 @@ cp ../../LICENSE Resources/LICENSE
 cp ../logos/alpha_logo.png Resources/alpha_logo.png
 chmod 777 scripts/*
 
-pkgbuild --root dist/directlfq --identifier de.mpg.biochem.directlfq.app --version 0.2.19 --install-location /Applications/directlfq.app --scripts scripts directlfq.pkg
+pkgbuild --root dist/directlfq --identifier de.mpg.biochem.directlfq.app --version 0.2.20 --install-location /Applications/directlfq.app --scripts scripts directlfq.pkg
 productbuild --distribution distribution.xml --resources Resources --package-path directlfq.pkg dist/directlfq_gui_installer_macos.pkg

release/one_click_macos_gui/distribution.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8" standalone="no"?>
 <installer-script minSpecVersion="1.000000">
- <title>directlfq 0.2.19</title>
+ <title>directlfq 0.2.20</title>
  <background mime-type="image/png" file="alpha_logo.png" scaling="proportional"/>
  <welcome file="welcome.html" mime-type="text/html" />
  <conclusion file="conclusion.html" mime-type="text/html" />

release/one_click_windows_gui/create_installer_windows.sh

@@ -17,7 +17,7 @@ python setup.py sdist bdist_wheel
 # Setting up the local package
 cd release/one_click_windows_gui
 # Make sure you include the required extra packages and always use the stable or very-stable options!
-pip install "../../dist/directlfq-0.2.19-py3-none-any.whl[stable, gui]"
+pip install "../../dist/directlfq-0.2.20-py3-none-any.whl[stable, gui]"
 
 # Creating the stand-alone pyinstaller folder
 pip install pyinstaller==4.10

release/one_click_windows_gui/directlfq_innoinstaller.iss

@@ -2,7 +2,7 @@
 ; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES!
 
 #define MyAppName "directlfq"
-#define MyAppVersion "0.2.19"
+#define MyAppVersion "0.2.20"
 #define MyAppPublisher "Max Planck Institute of Biochemistry and the University of Copenhagen, Mann Labs"
 #define MyAppURL "https://github.com/MannLabs/directlfq"
 #define MyAppExeName "directlfq_gui.exe"

settings.ini

@@ -13,7 +13,7 @@ author = Constantin Ammar
 author_email = [email protected]
 copyright = fast.ai
 branch = master
-version = 0.2.19
+version = 0.2.20
 min_python = 3.6
 audience = Developers
 language = English

tests/run_quicktests.sh

@@ -4,5 +4,6 @@ python download_testfiles.py quicktest
 cd quicktests
 jupyter nbconvert --to script run_pipeline_w_different_input_formats.ipynb
 python run_pipeline_w_different_input_formats.py
+directlfq lfq -i ../../test_data/system_tests/quicktests/diann/shortened_input.tsv
 cd ..
 conda deactivate