Rti merge (#28)

* tranql upgrade python/docker py to 3.10 * Update server.py to py3.10 yaml * Update tranql to redis-stack and add dev-conf and dev-schema files * Make conf.yml file configurable and add to README * Update requirements * updated docker compose to handle some errors with writing to redis and passing all values to tranql through env * fixing error produced by missing conf in tranql api call * fix uvicorn vulnerability issues related to Plater * remove all 'biolink:' additions in code * upgrade plater version to fix biolink issue * plater biolink changes * Tranql works with new biolink data model * updated Plater to new version * fixed plater version * bump version * force update fix, and flasgger version bump * bump old versions * more vul fixes, and version updates * down to zero vul --------- Co-authored-by: Nathan Braswell <[email protected]> Co-authored-by: esurface <[email protected]> Co-authored-by: braswent <[email protected]>
helxplatform · Oct 25, 2023 · bcddea0 · bcddea0
1 parent 10362c4
commit bcddea0
Show file tree

Hide file tree

Showing 23 changed files with 246 additions and 86 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,14 +1,10 @@
-FROM node:14.16.1-alpine as node
-FROM python:3.7.3-alpine
+FROM python:3.11.6-alpine3.18
+# FROM python:3.9.16-alpine3.17
+# FROM python:3.7.3-alpine OLD
 
-COPY --from=node /usr/lib /usr/lib
-COPY --from=node /usr/local/share /usr/local/share
-COPY --from=node /usr/local/lib /usr/local/lib
-COPY --from=node /usr/local/include /usr/local/include
-COPY --from=node /usr/local/bin /usr/local/bin
-
-RUN apk add --update build-base git libxslt-dev linux-headers make zeromq zeromq-dev
-RUN npm i -g [email protected]
+RUN apk add --update build-base git libxslt-dev linux-headers make nodejs-current npm zeromq zeromq-dev
+RUN apk upgrade libssl3 libcrypto3 
+# RUN apk add ncurses-libs=6.3_p20221119-r1
 
 ENV USER tranql
 ENV HOME /home/$USER
@@ -24,13 +20,12 @@ ENV BACKPLANE=http://tranql-backplane.renci.org
 
 COPY --chown=$USER . tranql/
 
-WORKDIR $HOME/tranql/src/tranql/web
-RUN npm install --legacy-peer-deps
-ENV NODE_OPTIONS=--max-old-space-size=4096
-ENV GENERATE_SOURCEMAP=false
-RUN npm run build
+# WORKDIR $HOME/tranql/src/tranql/web
+# RUN npm install
+# RUN GENERATE_SOURCEMAP=false npm run build
 
 WORKDIR $HOME/tranql
 RUN pip install --user --upgrade pip
 RUN pip install --user -r requirements.txt
+RUN pip uninstall -y py
 ENV PYTHONPATH=$HOME/tranql/src/
diff --git a/Dockerfile_Old b/Dockerfile_Old
@@ -0,0 +1,26 @@
+FROM python:3.7.3-alpine
+
+RUN apk add --update build-base git libxslt-dev linux-headers make nodejs-current nodejs-npm zeromq zeromq-dev
+
+ENV USER tranql
+ENV HOME /home/$USER
+
+RUN addgroup -S $USER && adduser -S $USER -G $USER -s /bin/bash -h $HOME
+
+USER $USER
+WORKDIR $HOME
+
+ENV PATH=$HOME/.local/bin:$PATH
+ENV PUBLIC_URL={{web_prefix}}
+ENV BACKPLANE=http://tranql-backplane.renci.org
+
+COPY --chown=$USER . tranql/
+
+WORKDIR $HOME/tranql/src/tranql/web
+RUN npm install
+RUN GENERATE_SOURCEMAP=false npm run build
+
+WORKDIR $HOME/tranql
+RUN pip install --user --upgrade pip
+RUN pip install --user -r requirements.txt
+ENV PYTHONPATH=$HOME/tranql/src/
diff --git a/README.md b/README.md
@@ -117,7 +117,7 @@ TranQL is brand new and strictly alpha.
 
 ### Install:
 
-Requires Python 3.7.x.
+Requires Python>=3.10
 
 ```
 git clone <repository>
@@ -128,17 +128,69 @@ pip install -r tranql/requirements.txt
 ```
 bin/test
 ```
-### Run
 
-To run a program, first start the backplane:
+### Local Dev Environment
+
+Run `TranQL` locally in a development environment. This set up uses the `./src/tranql/dev-conf.yml` configuration file
+and the `./src/tranql/conf/dev-schema.yml` file. Update those to edit your local configuration.
+
+1. Set up local environment variables
+```
+export REDIS_PASSWORD=<password>
+export SCHEMA_CONFIG_PATH=./src/tranql/conf/dev-schema.yml
+```
+
+2. Look at `dev-schema.yml` file
+    - Current default is to use `redis` as the database
+
+3. Install an editable build
+
+```
+pip install -e .
+```
+
+4. Choose your server
+    - Non-docker: 
+    To run a program, first start the backplane:
+        ```
+        cd backplane
+        PYTHONPATH=$PWD/../.. python server.py
+        ```
+    - Docker with redis
+        ```
+        docker-compose build redis
+        docker-compose up redis
+        ```
+5. Run a query
+
+```
+python main.py --source queries/workflow-5.tranql 
 ```
-cd backplane
-PYTHONPATH=$PWD/../.. python server.py
+
+### Local Docker Environment
+
+Run `TranQL` in a local docker environment using `docker-compose`. This will use the configurations 
+in the docker the `docker-compose.yml` file 
+
+1. Set the local environment variables
+```
+export REDIS_PASSWORD=<password>
+export SCHEMA_CONFIG_PATH=./src/tranql/conf/redis-schema.yml
+```
+
+2. Build and start the docker containers
 ```
-Then run the query:
+docker-compose build
+docker-compose up
 ```
-bin/tranql --source tranql/queries/workflow-5.tranql
+
+3. Run a query
+
+```
+python main.py --source queries/workflow-5.tranql 
 ```
+
+
 ### Web app
 
 To run the web app, first start the TranQL API:
@@ -149,6 +201,7 @@ PYTHONPATH=$PWD/../ python api.py
 ```
 
 Then follow the instructions in web/ to start the website.
+
 ### Shell
 
 Run the interactive interpreter.

diff --git a/Vulnerabilities_Doc.md b/Vulnerabilities_Doc.md
@@ -0,0 +1,20 @@
+# Tranql Vulnerabilities
+
+## tranql/Requirements.txt file
+
+### Pip packages
+
+- flask-cors: High 3.0.7 => fix 3.0.9
+- jinja2: High 2.10 => fix 2.10.1
+- pyyaml: 3 CRITICAL 5.1 => Test newest version at 6.0
+
+### Git repositories
+
+- git+<https://github.com/helxplatform/[email protected]>
+- git+<https://github.com/TranslatorSRI/[email protected]#egg=reasoner-pydantic>
+- git+<https://github.com/TranslatorSRI/[email protected]#egg=reasoner-converter>
+
+## tranql/src/tranql/web/package.json file
+
+Completely broken... cannot build locally.
+Tons of conflicting dependencies... checking with Renci if UI is even used.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -7,12 +7,23 @@ services:
     ports:
       - "8001:8001"
     command: gunicorn --workers=1 --bind=0.0.0.0:8001 --name=tranql --timeout=600 --reload tranql.api:app
-    environment:
-      - SCHEMA_CONFIG_PATH=/home/tranql/dev-schema.yaml
+    env_file: .env
     volumes:
       - ./src:/home/tranql/tranql/src
-      - ./dev-schema.yaml:/home/tranql/dev-schema.yaml
-  redis_graph:
-    image: redislabs/redisgraph
+      - ./src/tranql/conf/redis-schema.yml:/home/tranql/schema.yaml
+
+  #################################################################################
+  ##
+  ## A memory cache for results of high volume service requests.
+  ## https://redis.io/docs/stack/get-started/install/docker/
+  ##
+  #################################################################################
+  redis:
+    image: 'redis/redis-stack:6.2.4-v2'
+    environment:
+      - REDIS_ARGS=--requirepass $REDIS_PASSWORD
+      - REDIS_DISABLE_COMMANDS=FLUSHDB,FLUSHALL
+    ports:
+      - '6379:6379'
     volumes:
       - ./redis_data:/data
diff --git a/redis_data/dump.rdb b/redis_data/dump.rdb
diff --git a/requirements.txt b/requirements.txt
@@ -1,30 +1,35 @@
-aiohttp==3.5.4
+aiohttp==3.8.6
+bmt==0.8.12
 coverage==4.5.3
-flasgger==0.9.2
-flask==1.0.2
-flask-cors==3.0.7
-flask-restx==0.5.1
-gunicorn==19.9.0
-MarkupSafe==2.0.1
-itsdangerous==2.0.1
-jinja2==2.10
+flasgger==0.9.5
+Flask==2.2.5
+Flask-Compress==1.13
+Flask-Cors==3.0.10
+flask-restx>=1.0.3
+gunicorn==21.2.0
+Jinja2==3.1.2
 jsonpath-rw==1.4.0
-networkx~=2.5.0
+linkml==1.4.6
+linkml-dataops==0.1.0
+linkml-runtime==1.3.2
+MarkupSafe==2.1.1
 openapi==1.1.0
+git+https://github.com/RTIInternational/[email protected]
+git+https://github.com/TranslatorSRI/[email protected]#egg=reasoner-pydantic
+git+https://github.com/TranslatorSRI/[email protected]#egg=reasoner-converter
+reasoner-transpiler==1.7.1
+redis==4.1.4
+requests==2.31.0
+requests-cache==0.4.13
+requests-mock==1.5.2
+PyYAML==6.0.1
+python-Levenshtein==0.12.2
+networkx==2.8.8
 pytest==5.4.1
 pytest-cov==2.7.1
 python-coveralls==2.9.2
 python-dateutil==2.8.1
-pyyaml==5.1
-requests==2.25.1
-requests-cache==0.4.13
-requests-mock==1.5.2
-git+https://github.com/helxplatform/[email protected]
-bmt==0.8.2
-linkml==1.1.17
-linkml-dataops==0.1.0
-linkml-runtime==1.1.21
-reasoner-transpiler==1.7.1
-python-levenshtein==0.12.2
-git+https://github.com/TranslatorSRI/[email protected]#egg=reasoner-pydantic
-git+https://github.com/TranslatorSRI/[email protected]#egg=reasoner-converter
+redis==4.1.4
+werkzeug==2.2.3
+urllib3==1.26.17
+uvicorn
diff --git a/requirements_Old.txt b/requirements_Old.txt
@@ -0,0 +1,30 @@
+aiohttp==3.5.4
+coverage==4.5.3
+flasgger==0.9.2
+flask==1.0.2
+flask-cors==3.0.7
+flask-restx==0.5.1
+gunicorn==19.9.0
+MarkupSafe==2.0.1
+itsdangerous==2.0.1
+jinja2==2.10
+jsonpath-rw==1.4.0
+networkx~=2.5.0
+openapi==1.1.0
+pytest==5.4.1
+pytest-cov==2.7.1
+python-coveralls==2.9.2
+python-dateutil==2.8.1
+pyyaml==5.1
+requests==2.25.1
+requests-cache==0.4.13
+requests-mock==1.5.2
+git+https://github.com/helxplatform/[email protected]
+bmt==0.8.2
+linkml==1.1.17
+linkml-dataops==0.1.0
+linkml-runtime==1.1.21
+reasoner-transpiler==1.7.1
+python-levenshtein==0.12.2
+git+https://github.com/TranslatorSRI/[email protected]#egg=reasoner-pydantic
+git+https://github.com/TranslatorSRI/[email protected]#egg=reasoner-converter
diff --git a/setup.cfg b/setup.cfg
@@ -19,7 +19,7 @@ classifiers =
 package_dir =
     = src
 packages = find:
-python_requires = >=3.7
+python_requires = 3.10
 include_package_data = true
 install_requires =
     Flask

diff --git a/src/tranql/_version.py b/src/tranql/_version.py
@@ -1 +1 @@
-__version__ = "0.4.dev5"
+__version__ = "1.0.dev0"
diff --git a/src/tranql/api.py b/src/tranql/api.py
@@ -69,7 +69,7 @@ def base_path(self):
 definitions_filename = os.path.join(os.path.dirname(__file__), definitions_filename)
 with open(filename, 'r') as file_obj:
     template = {
-        "definitions": yaml.load(file_obj)["definitions"],
+        "definitions": yaml.full_load(file_obj)["definitions"],
         "tags": [
             {"name": "query"},
             {"name": "schema"},
@@ -79,7 +79,7 @@ def base_path(self):
         ]
     }
     with open(definitions_filename, 'r') as definitions_file:
-        template['definitions'].update(yaml.load(definitions_file))
+        template['definitions'].update(yaml.full_load(definitions_file))
 
 swagger = Swagger(app, template=template, config={
     "headers": [
@@ -549,7 +549,7 @@ def get(self):
               default: false
               description: Specifies if dynamic id lookup of curies will be performed
         """
-        force_update = request.args.get("force_update")
+        force_update = request.args.get("force_update").lower() == 'true'
         tranql = TranQL (options={"registry": app.config.get('registry', False)})
         schemafactory = tranql.schema_factory
         schema = schemafactory.get_instance(force_update=force_update)
@@ -670,7 +670,7 @@ def post(self):
                 type: string
               allowed_concept_types:
                 description: >
-                  List of allowed biolink types to search against, e.g. biolink:Disease.
+                  List of allowed biolink types to search against, e.g. biolink.Disease.
                   If null or empty, the search will execute against all supported types.
                   Note that if searching against all types, a very high `query_limit` should be specified
                   since it will be distributed evenly across every supported concept index in the redisgraph
@@ -707,7 +707,7 @@ def post(self):
     """
     query = request.json["query"]
     # Indexes are equal to node labels in the redisgraph instance,
-    # and each node label is simply a biolink concept type, e.g. biolink:Disease
+    # and each node label is simply a biolink concept type, e.g. biolink.Disease
     indexes = request.json.get("allowed_concept_types", None)
     fields = request.json.get("fields", None)
     prefix_search = request.json.get("prefix_search", True)
@@ -728,7 +728,7 @@ def post(self):
 
     if indexes is None or len(indexes) == 0:
       concept_types = schema.schema[redis_schema_name]["schema"].keys()
-      indexes = ["biolink:" + title_case(concept_type) for concept_type in concept_types]
+      indexes = ["biolink." + title_case(concept_type) for concept_type in concept_types]
 
 
     return redis_adapter.search(
@@ -739,8 +739,8 @@ def post(self):
       options={
         "prefix_search": prefix_search,
         # Ensure results are linked to studies
-        "postprocessing_cypher": "MATCH (:`biolink:StudyVariable`)-[]-(node)" if study_linked else "",
-        # "postprocessing_cypher": "MATCH ()-[:`biolink:Association`|`biolink:association`|`biolink:Mentions`|`biolink:mentions`]->(node)" if study_linked else "",
+        "postprocessing_cypher": "MATCH (:`biolink.StudyVariable`)-[]-(node)" if study_linked else "",
+        # "postprocessing_cypher": "MATCH ()-[:`biolink.Association`|`biolink.association`|`biolink.Mentions`|`biolink.mentions`]->(node)" if study_linked else "",
         "levenshtein_distance": levenshtein_distance,
         "query_limit": query_limit
       }

diff --git a/src/tranql/backplane/api/icees_api.py b/src/tranql/backplane/api/icees_api.py
@@ -238,13 +238,13 @@ def get_supported_type(self):
 
     def curify_type(self, concept_type_str):
         """
-        Converts to biolink:TypeOfConcept from type_of_concept
+        Converts to biolink.TypeOfConcept from type_of_concept
         :param concept_type_str
-        :return: pascal case of type with prefix biolink:
+        :return: pascal case of type with prefix biolink.
         """
-        if concept_type_str.startswith('biolink:'):
+        if concept_type_str.startswith('biolink.'):
             return concept_type_str
-        return 'biolink:' + string.capwords(concept_type_str.replace('_', ' '), ' ').replace(' ', '')
+        return string.capwords(concept_type_str.replace('_', ' '), ' ').replace(' ', '')
 
     def synonymize(self, response):
         knowledge_map_key = 'knowledge_map'