Tidy up

add ruff for linting and formatting
set up pre-commit
apply pre-commit
clean up code (docker, server) before XML conversion

.docker/_activate_current_env.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+source /opt/conda/etc/profile.d/conda.sh
+conda activate osm
+exec "$@"

.docker/_apptainer_shell.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+source ${MAMBA_ROOT_PREFIX}/etc/profile.d/conda.sh
+conda activate ${ENV_NAME}
+exec "\$@"

.docker/_dockerfile_initialize_user_accounts.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+addgroup --gid ${MAMBA_USER_GID} ${MAMBA_USER}
+adduser --uid ${MAMBA_USER_ID} --gid ${MAMBA_USER_GID} --disabled-password --gecos "" ${MAMBA_USER}

.docker/_dockerfile_setup_root_prefix.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+mkdir -p ${MAMBA_ROOT_PREFIX}
+chown -R ${MAMBA_USER}:${MAMBA_USER} ${MAMBA_ROOT_PREFIX}

.docker/_dockerfile_shell.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+eval $(/opt/conda/bin/conda shell.bash hook)
+conda activate osm
+exec "$@"

.docker/_entrypoint.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+source /opt/conda/etc/profile.d/conda.sh
+conda activate osm
+exec "$@"

.github/workflows/tests.yml

@@ -6,47 +6,14 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v3
     - uses: pre-commit/[email protected]
-  tox-tests:
+  pytest-suite:
     runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        python-version: [3.9]
-
     steps:
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Cache pip
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -e .[dev]
-
-      - name: Start ScienceBeam Docker container
-        run: |
-          docker run -d --rm -p 8070 elifesciences/sciencebeam-parser
-
-      - name: Run tests
-        run: |
-          tox
-
-      - name: Test packaging
-        run: |
-          tox -e .package
+      - uses: actions/checkout@v2
+      - name: Build the stack
+        run: docker-compose up --build
+      - name: Run pytest
+        run: docker-compose run --entrypoint '/bin/bash -lc' app 'pytest tests'

.pre-commit-config.yaml

@@ -6,10 +6,13 @@ repos:
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: "v0.4.9"
+    rev: v0.5.0
     hooks:
+      # Run the linter.
       - id: ruff
         args: ["--fix"]
+      # Run the formatter.
+      - id: ruff-format
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0

Dockerfile

@@ -0,0 +1,42 @@
+FROM condaforge/mambaforge:24.3.0-0
+SHELL ["/bin/bash", "--login", "-c"]
+# Set working directory
+WORKDIR /app
+
+# Install debugging tools
+RUN apt-get update && apt-get install -y \
+    git \
+    curl \
+    iputils-ping \
+    net-tools \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy project files for installation
+ENV ENV_NAME=osm
+COPY environment.yaml /app
+
+# Create the environment
+RUN conda env create -f environment.yaml
+
+# Ensure the conda environment is activated
+RUN echo "source /opt/conda/etc/profile.d/conda.sh && conda activate osm" | tee -a ~/.bashrc /etc/profile /etc/profile.d/conda.sh /etc/skel/.bashrc /etc/skel/.profile > /dev/null
+
+RUN R -e '\
+install.packages("roadoi", repos = "http://cran.us.r-project.org"); \
+devtools::install_github("quest-bih/oddpub"); \
+devtools::install_github("cran/crminer"); \
+devtools::install_github("serghiou/metareadr"); \
+devtools::install_github("serghiou/rtransparent")'
+
+# Copy the project files and install the package
+COPY pyproject.toml /app
+COPY osm /app/osm
+COPY .git /app/.git
+
+# Install the package in editable mode
+RUN pip install -e .
+
+# Make entrypoint etc. convenient for users
+COPY .docker/_entrypoint.sh /usr/local/bin/_entrypoint.sh
+ENTRYPOINT ["/usr/local/bin/_entrypoint.sh", "--", "osm"]
+CMD ["--help"]

README.md

@@ -1,22 +1,66 @@
 # OpenSciMetrics
+
 OpenSciMetrics (OSM) applies NLP and LLM-based metrics and indicators related to transparency, data sharing, rigor, and open science on biomedical publications.
 
-# How to setup and run the application
-- After cloning the repo, navigate into the project's root directory by running `cd osm`
-- Run `python -m venv venv` to create a Virtual Environment
-- Depending on your system, run the approriate command to Activate the Virtual Environment
-Windows: `venv\Scripts\activate`<br>
-macOS and Linux: `source venv/bin/activate`
+# Running the app
+
+- From the project's root directory run:
+
+```
+docker-compose -f compose.yaml run \
+  --rm \
+  -v $PWD:/mnt \
+  app \
+  rtransparent \
+  /mnt/docs/examples/pdf_inputs/test_sample.pdf \
+  output.xml
+```
+
+
+# Development
+
+You can use docker-compose for development with the local source code mounted into the app container (this uses compose.override.yaml):
+
+```
+docker-compose up
+```
+
+If you are changing dependencies or the docker files you will want to rebuild the images:
+
+```
+docker-compose up --build
+```
+
+Troubleshooting issues with the osm application can be done with:
+
+```
+docker-compose run --entrypoint bash app
+```
+
+You can also set up this package for local development (on Linux x86 architecture only) by cloning the repo, navigate into the project's root directory, and follow the steps in the [Dockerfile](./Dockerfile) to:
+- create a conda environment
+- install R dependencies only available from github/CRAN
+- install the package in editable mode
+
+Open a terminal tab and run the image `docker run --rm -p 8070 elifesciences/sciencebeam-parser` and keep it running (or use docker-compose as described above).
+
+**Note:** The ScienceBeam image  is not supported on apple silicon chips  with emulation  (using --platform=linux/amd64). The conda environment also doesn't solve at the moment due to various missing packages on this architecture.
+
+
+## To test that the sciencebeam server is working:
+
+This will work if you have exposed the port 8070 from the sciencebeam container (done for development in the docker override file):
+```
+curl --fail --show-error     --form "file=@docs/examples/pdf_inputs/test_sample.pdf;filename=test_sample.pdf"     --silent "http://localhost:8070/api/pdfalto"
+```
+
+You can alter the above command to work from other containers if you use app_network with the container and target the host "sciencebeam".
 
-- Next, run `pip install -e .` to install the package with its dependencies.
-- Finally, run `osm pdf-xml "path_to_file_name.pdf" file_id`
+## How to run the tests
 
-# How to run tests of the application
-Run `tox`
-# How to run the unit tests
 - Navigate to the project's root directory and run `pytest`
 
-# Using pre-commit for commit checks
+## Using pre-commit for commit checks
 
 Pre-commit will run all of its hooks on every commit you make. To install
 pre-commit and its hooks, run the following commands:

compose.override.yaml

@@ -0,0 +1,9 @@
+services:
+  sciencebeam:
+    ports:
+      # Forward the port 8070 on the host to the port 8070 on the container for
+      # convenience during development
+      - "8070:8070"
+  app:
+    volumes:
+      - .:/app

compose.yaml

@@ -0,0 +1,18 @@
+services:
+  sciencebeam:
+    image: elifesciences/sciencebeam-parser
+    networks:
+      - app_network
+
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      - sciencebeam
+    networks:
+      - app_network
+
+networks:
+  app_network:
+    driver: bridge

environment.yaml

@@ -0,0 +1,54 @@
+name: osm
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  - python=3.11
+  - click
+  - python-dotenv
+  - ipython
+  - lxml
+  - pandas
+  - pip
+  - pkginfo
+  - pre-commit
+  - pytest
+  - pytest-cov
+  - pytest-mock
+  - requests
+  - rich
+  - rpy2
+  - ruff=0.5.0
+  # Dependencies for rtransparent
+  - r-crul
+  - r-devtools
+  - r-dplyr
+  - r-furrr
+  - r-future
+  - r-globals
+  - r-hoardr
+  - r-httpcode
+  - r-lazyeval
+  - r-lubridate
+  - r-magrittr
+  - r-pbapply
+  - r-pdftools
+  - r-plyr
+  - r-purrr
+  - r-qpdf
+  - r-readr
+  - r-rentrez
+  - r-rlang
+  - r-stringr
+  - r-tibble
+  - r-tidyr
+  - r-tidyselect
+  - r-timechange
+  - r-tokenizers
+  - r-triebeard
+  - r-urltools
+  - r-utf8
+  - r-XML
+  - r-xml2
+  - pip:
+    - metapub

osm/__init__.py

@@ -1,3 +1,27 @@
-from . import _version
+import os
 
-__version__ = _version.version
+
+def get_version():
+    try:
+        from . import _version
+
+        return _version.version
+    except ImportError:
+        generate_version_file()
+        from . import _version
+
+        return _version.version
+
+
+def generate_version_file():
+    import pkg_resources
+
+    version = pkg_resources.get_distribution("osm").version
+    version_file_content = f"version = '{version}'\n"
+
+    version_file_path = os.path.join(os.path.dirname(__file__), "_version.py")
+    with open(version_file_path, "w") as version_file:
+        version_file.write(version_file_content)
+
+
+__version__ = get_version()

osm/cli.py

@@ -0,0 +1,17 @@
+import click
+
+from osm.converters import convert_pdf
+
+
+@click.group()
+def osm():
+    """Main command for OSM"""
+    pass
+
+
+@osm.command()
+@click.argument("file_path", type=click.Path(exists=True))
+@click.argument("output_file", type=str)
+def rtransparent(file_path, output_file):
+    """Processes a biomedical publication. Writes out processed document and associated metrics."""
+    convert_pdf(file_path, output_file)