Merge pull request #28 from moevm/26-import-export-dump

#26 - Импорт и экспорт данных

.gitignore

@@ -1,2 +1,3 @@
 **/__pycache__/
-**/venv/
+**/venv/
+./backend/app/models/dumps/dump.csv

backend/app/models/utils/allowedEntity.py

@@ -0,0 +1,64 @@
+allowed_entity_parameters = {
+    "Patient": [
+        "fullname",
+        "mail",
+        "password",
+        "sex",
+        "birthday",
+        "last_update",
+        "registration_date",
+        "height",
+        "weight",
+        "admin"
+    ],
+    "Appeal": [
+        "appeal_date",
+        "appeal_complaints"
+    ],
+    "Symptom": [
+        "symptom_name",
+        "symptom_description"
+    ],
+    "Disease": [
+        "disease_name",
+        "disease_description",
+        "disease_recommendations",
+        "disease_type",
+        "disease_course"
+    ],
+    "Analysis": [
+        "disease_name",
+        "disease_source"
+    ]
+}
+
+allowed_relations = ['Symptom-Disease', 'Appeal-Disease', 'Symptom-Analysis', 'Patient-Appeal', 'Appeal-Symptom']
+
+CSV_columns = [
+    "type",
+    "fullname",
+    "mail",
+    "password",
+    "sex",
+    "age",
+    "height",
+    "weight",
+    "admin",
+    "birthday",
+    "last_update",
+    "registration_data",
+    "disease_name",
+    "disease_description",
+    "disease_recommendations",
+    "disease_type",
+    "disease_course",
+    "symptom_name",
+    "symptom_description",
+    "analysis_name",
+    "analysis_source",
+    "relation_from",
+    "relation_to",
+    "symptom_weight",
+    "appeal_date",
+    "appeal_complaints"
+]

backend/app/models/utils/modelsForDumpTransform.py

@@ -0,0 +1,27 @@
+def create_relation_dict(r_from, r_to, r, r_type):
+    new_dict = {}
+    new_dict["type"] = r_type
+
+    match r_type:
+        case "Symptom-Disease":
+            new_dict["relation_from"] = r_from["symptom_name"]
+            new_dict["relation_to"] = r_to["disease_name"]
+            new_dict["symptom_weight"] = r
+
+        case "Appeal-Disease":
+            new_dict["relation_from"] = r_from["appeal_date"]
+            new_dict["relation_to"] = r_to["disease_name"]
+
+        case "Symptom-Analysis":
+            new_dict["relation_from"] = r_from["symptom_name"]
+            new_dict["relation_to"] = r_to["analysis_name"]
+
+        case "Patient-Appeal":
+            new_dict["relation_from"] = r_from["mail"]
+            new_dict["relation_to"] = r_to["appeal_date"]
+
+        case "Appeal-Symptom":
+            new_dict["relation_from"] = r_from["appeal_date"]
+            new_dict["relation_to"] = r_to["symptom_name"]
+
+    return new_dict 

backend/app/routes.py

@@ -1,11 +1,13 @@
 from app import app
 from app.models.neo4jConnection import Neo4jConnection
-from app.models.allowedEntity import allowed_entity_parameters
+from app.models.utils.allowedEntity import allowed_entity_parameters, CSV_columns, allowed_relations
+from app.models.utils.modelsForDumpTransform import create_relation_dict
 import re
 from datetime import datetime
 from flask import Flask, render_template, request, redirect, url_for, session, Response, jsonify, json
 import requests
 import os
+import csv 
 
 uri = os.getenv("NEO4J_URI", "bolt://localhost:7687")
 user = os.getenv("NEO4J_USER", "neo4j")
@@ -42,9 +44,9 @@ def register() -> str:
                                     'weight' in request.form and \
                                     "admin" in request.form:
 
-        full_name : str = request.form['full_name']
+        fullname : str = request.form['full_name']
         password : str = request.form['password']
-        email : str = request.form['email']
+        mail : str = request.form['email']
         sex : str = request.form['sex']
         birthday : str = request.form['birthday']
         height : float = request.form['height']
@@ -69,10 +71,10 @@ def register() -> str:
 
         else:
             query_string = '''
-            MERGE (p:Patient {full_name: $full_name, password: $password, email: $email, sex: $sex, birthday: $birthday, height: $height, weight: $weight, registration_date: $rd, admin: $admin})
+            MERGE (p:Patient {fullname: $fullname, password: $password, mail: $email, sex: $sex, birthday: $birthday, height: $height, weight: $weight, registration_date: $rd, admin: $admin})
             '''
 
-            conn.query(query_string, {"full_name": full_name, "password": password, "email": email,
+            conn.query(query_string, {"fullname": fullname, "password": password, "mail": email,
                                     "sex": sex, "birthday": birthday, "rd": datetime.now().isoformat(), "height": height, "weight": weight, "admin": admin})
 
             msg = "Success"
@@ -107,8 +109,8 @@ def login() -> str:
         if patient: 
             patient_data : dict = patient[0].data()["p"]
             session["loggedin"] = True
-            session["email"] = patient_data["email"]
-            session["full_name"] = patient_data["full_name"]
+            session["mail"] = patient_data["mail"]
+            session["fullname"] = patient_data["fullname"]
             session["admin"] = patient_data["admin"]
         else:
             msg = 'Неправильный логин или пароль'
@@ -252,3 +254,114 @@ def db_page(entity_type):
     return render_template('data_bases.html', session = session, certain_page = False, entity_type = entity_type, lst = data)
 
 
+@app.route('/import_dump', methods=['POST'])
+def import_dump():
+    query_strings : list(str) = [
+        '''
+        LOAD CSV WITH HEADERS FROM 'file:///dump.csv' AS row
+        WITH row WHERE row.type = 'Patient'
+        MERGE (p:Patient {fullname: row.fullname, mail: row.mail, password: row.password, sex: row.sex, age: row.age, height: row.height, weight: row.weight, last_update: row.last_update, admin: row.admin, birthday: row.birthday, registration_data: row.registration_data});
+        ''',
+
+        '''
+        LOAD CSV WITH HEADERS FROM 'file:///dump.csv' AS row
+        WITH row WHERE row.type = 'Symptom'
+        MERGE (s:Symptom {symptom_name: row.symptom_name, symptom_description: row.symptom_description});
+        ''',
+
+        '''
+        LOAD CSV WITH HEADERS FROM 'file:///dump.csv' AS row
+        WITH row WHERE row.type = 'Disease'
+        MERGE (d:Disease {disease_name: row.disease_name, disease_description: row.disease_description, disease_recommendations: row.disease_recommendations, disease_type: row.disease_type, disease_course: row.disease_course});
+        ''',
+
+        '''
+        LOAD CSV WITH HEADERS FROM 'file:///dump.csv' AS row
+        WITH row WHERE row.type = 'Analysis'
+        MERGE (an:Analysis {analysis_name: row.analysis_name, analysis_source: row.analysis_source});
+        ''',
+
+        '''
+        LOAD CSV WITH HEADERS FROM 'file:///dump.csv' AS row
+        WITH row WHERE row.type = 'Appeal'
+        MERGE (ap:Appeal {appeal_date: row.appeal_date, appeal_complaints: row.appeal_complaints});
+        ''',
+
+        '''
+        LOAD CSV WITH HEADERS FROM 'file:///dump.csv' AS row
+        WITH row WHERE row.type = 'Patient-Appeal'
+        MATCH (p:Patient {mail: row.relation_from}), (a:Appeal {appeal_date: row.relation_to})
+        MERGE (p)-[:create]->(a)
+        MERGE (a)-[:belong]->(p);
+        ''',
+
+        '''
+        LOAD CSV WITH HEADERS FROM 'file:///dump.csv' AS row
+        WITH row WHERE row.type = 'Appeal-Symptom'
+        MATCH (a:Appeal {appeal_date: row.relation_from}), (s:Symptom {symptom_name: row.relation_to})
+        MERGE (a)-[:contain]->(s);
+        ''',
+
+        '''
+        LOAD CSV WITH HEADERS FROM 'file:///dump.csv' AS row
+        WITH row WHERE row.type = 'Symptom-Analysis'
+        MATCH (s:Symptom {symptom_name: row.relation_from}), (an:Analysis {analysis_name: row.relation_to})
+        MERGE (s)-[:confirm]->(an);
+        ''',
+
+        '''
+        LOAD CSV WITH HEADERS FROM 'file:///dump.csv' AS row
+        WITH row WHERE row.type = 'Appeal-Disease'
+        MATCH (a:Appeal {appeal_date: row.relation_from}), (d:Disease {disease_name: row.relation_to})
+        MERGE (a)-[:predict]->(d);
+        ''',
+
+        '''
+        LOAD CSV WITH HEADERS FROM 'file:///dump.csv' AS row
+        WITH row WHERE row.type = 'Symptom-Disease'
+        MATCH (d:Disease {disease_name: row.relation_from}), (s:Symptom {symptom_name: row.relation_to})
+        MERGE (s)-[:describe {symptom_weight: row.symptom_weight}]->(d)
+        MERGE (d)-[:cause {symptom_weight: row.symptom_weight}]->(s);
+        '''
+    ]
+
+    for query_string in query_strings:     
+        result = conn.query(query_string)
+        if result is None:
+            return jsonify({"Error": f"error loading the database dump: {query_string}"}), 400
+
+    return "Success"
+
+@app.route('/export_dump', methods=['POST'])
+def export_dump():
+    file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'models/dumps/dump.csv'))
+
+    with open(file_path, "w") as csvfile:
+        writer = csv.DictWriter(csvfile, delimiter = ',', quotechar = '"', quoting = csv.QUOTE_ALL, fieldnames=CSV_columns)
+        writer.writeheader()
+
+        for entity_type in allowed_entity_parameters.keys():
+            response = requests.post("http://127.0.0.1:5000/entities", data={'entity_type': entity_type})
+            data = response.json()
+
+            for row in data:
+                row["type"] = entity_type
+                writer.writerow(row)
+
+        for relation_type in allowed_relations:
+            relation_from, relation_to = relation_type.split('-')
+
+            query_string : str = f'''
+            MATCH((a:{relation_from})-[r]->(b:{relation_to}))
+            RETURN a,r,b 
+            '''
+
+            relations_list : list[Record] = conn.query(query_string)
+
+            if relations_list:
+                for relation in relations_list:
+                    row_dict = create_relation_dict(relation.data()["a"], relation.data()["b"], relation['r'].get('symptom_weight', None), relation_type)
+                    writer.writerow(row_dict)
+
+    return "Success"
+

docker-compose.yml

@@ -2,7 +2,9 @@ version: '3.8'
 
 services:
   neo4j:
-    image: neo4j:5.25.1
+    build:
+      context: ./neo4j
+      dockerfile: Dockerfile
     ports:
       - "7474:7474"
       - "7687:7687"
@@ -29,12 +31,11 @@ services:
     networks:
       - neo4j-network
     volumes:
-      - backend-data:/backend
+      - ./backend/app/models/dumps:/backend/app/models/dumps
 
 networks:
   neo4j-network:
     driver: bridge
 
 volumes:
-  neo4j-data:
-  backend-data:
+  neo4j-data:

neo4j/Dockerfile

@@ -0,0 +1,9 @@
+FROM neo4j:5.25.1
+
+WORKDIR /var/lib/neo4j/import
+
+COPY dumps/ /var/lib/neo4j/import/
+
+ENV NEO4J_AUTH=neo4j/password
+
+EXPOSE 7474 7687