Merge branch 'develop' into main

src/.gitignore

@@ -0,0 +1,2 @@
+*.csv
+*.xlsx

src/main_generator_notebook.ipynb

@@ -19,9 +19,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n"
+     ]
+    }
+   ],
    "source": [
     "%load_ext autoreload\n",
     "%autoreload 2\n",
@@ -39,7 +48,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 7,
    "metadata": {
     "scrolled": false
    },
@@ -54,7 +63,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "76a0cbe43fda44eea61c8b089ca652ff",
+       "model_id": "22a268bbb4434226be2a3594dc00c670",
        "version_major": 2,
        "version_minor": 0
       },
@@ -68,7 +77,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "78544fc5fdb648c99ccc86097ae14583",
+       "model_id": "a5a91549f8784bc5bc2bbdd59dc8d628",
        "version_major": 2,
        "version_minor": 0
       },
@@ -82,7 +91,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "525292f510c44f918d15120885f25785",
+       "model_id": "f493109144654010b007ce3224b0a9ff",
        "version_major": 2,
        "version_minor": 0
       },
@@ -96,7 +105,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2db6b58d171b4e13b8df3541457d6582",
+       "model_id": "864fc73d44f5491f8b240b5dad54d9e4",
        "version_major": 2,
        "version_minor": 0
       },
@@ -130,27 +139,6 @@
     "10. https://www.bbc.com/news/health-45878325\n",
     "11. Fildes, A., Charlton, J., Rudisill, C., Littlejohns, P., Prevost, A. T., & Gulliford, M. C. (2015). Probability of an Obese Person Attaining Normal Body Weight: Cohort Study Using Electronic Health Records. American journal of public health, 105(9), e54–e59. https://doi.org/10.2105/AJPH.2015.302773"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'/home/victor/Documents/Expectation_data_generation/src'"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import os \n",
-    "os.getcwd()"
-   ]
   }
  ],
  "metadata": {

src/synthetic_data_generation/default_inputs.py

@@ -434,15 +434,15 @@
 }
 
 flexi_probabilities_dict = {
-    "flexi_vegie": {
+    "flexi_vegan": {
         "vegan_observant": 0.60,
         "vegetarian_observant": 0.20,
         "halal_observant": 0.10,
         "kosher_observant": 0.1,
         "None": 0.0
     },
     "flexi_vegetarian": {
-        # no meanginful flexi for this class:
+        # no meaningful flexi for this class:
         # vegetarian -> vegan
         "vegan_observant": 0.00,
         "vegetarian_observant": 0.60,
@@ -487,7 +487,7 @@
     "alone": 0.3,
     "family": 0.3,
     "friends": 0.2,
-    "colleges": 0.2
+    "colleagues": 0.2
 }
 
 delta_distribution_dict = {

src/synthetic_data_generation/generators.py

@@ -1,11 +1,14 @@
 import pandas as pd
 import numpy as np
+import datetime as dt
+import time
 from faker import Faker
 from enum import Enum
 from typing import List, Any, Tuple, Dict, Union, Set
 from functools import partial
 import string
 import os
+import json
 import uuid
 from parfor import parfor
 import seaborn as sns
@@ -1394,7 +1397,6 @@ def create_a_summary_table(df_total_user: pd.DataFrame,
     :rtype: HTML_Table
     """
     # health conditions
-    # TODO: update to work with the new format
     try:
         conditions = [BMI_constants.underweight, BMI_constants.healthy,
                       BMI_constants.overweight, BMI_constants.obesity]
@@ -1681,6 +1683,10 @@ def save_outputs(base_path: str, output_folder: str, files: Dict[str, Any]):
             # print(f"key: {k}, extension: {k.split('.')[-1]}")
             if k.split(".")[-1] == "csv":
                 files[k].to_csv(os.path.join(target_path, k))
+            elif k.split(".")[-1] == "npy":
+                # save parameters 
+                save_path = os.path.join(target_path, k)
+                np.save(save_path, files[k])
             elif k.split(".")[-1] == "png":
                 files[k].render(filename=os.path.join(
                     target_path, k), format='png')
@@ -1739,6 +1745,29 @@ def run_full_simulation(num_users: int,
     :return: Tuple with Tracking DataFrame, Users join DataFrame, and Summary HTML table.
     :rtype: Tuple[pd.DataFrame, pd.DataFrame, HTML_Table]
     """
+    # Save simulation parameters 
+    simulation_parameters = {
+                'simulation_date': dt.datetime.now().strftime('%d-%m-%Y_%H-%M-%S'),
+                "total_users": num_users,
+                "simulation_days": num_days,
+                'age_ranges': age_probabilities,
+                'gender': gender_probabilities,
+                'bmi': BMI_probabilities,
+                'allergies': allergies_probability_dict,
+                'cultural_restriction': food_restriction_probability_dict, 
+                'meal_probabilities': meals_proba, 
+                'flexible_probabilities': flexi_probabilities,
+                'bmi_transition_probabilities': probability_transition_matrix,
+                'meal_time': meals_time_dict, 
+                'meals_probability': meals_proba,
+                'place_of_meal_consumption': place_probabilities,
+                'social_situation_of_meal_consumption': social_situation_probabilities,
+                'appreciation_feedback': delta_dist_dict,
+                "num_simultaneous_allergies": multiple_allergies_number,
+                "delta_dist_chose": chose_dist,
+    }
+    # count time 
+    start_time = time.time()
     # Generate user data
     df_personal_data = generate_personal_data(num_users=num_users,
                                               age_probabilities=age_probabilities,
@@ -1804,4 +1833,8 @@ def run_full_simulation(num_users: int,
     # Create a summary table
     table = create_a_summary_table(df_user_join, new_tracking_df, df_recipes)
     # return the files
-    return df_user_join, table, new_tracking_df
+    end_time = time.time() - start_time
+    # print(f"Simulation finished in {end_time} seconds")
+    if progress_bar:
+        progress_bar.update_tail_text(f"Simulation finished in {np.round(end_time, 3)} seconds")
+    return df_user_join, table, new_tracking_df, simulation_parameters

src/visualization/visualization_helpers.py

@@ -13,6 +13,7 @@
 import graphviz as graphv
 import traceback
 from functools import partial
+import io
 
 from synthetic_data_generation.generators import (person_entity,
                                                   HTML_Table,
@@ -85,10 +86,12 @@ def __init__(self, initial_value: float = 0.0,
                  step=1.0,
                  description='generating...',
                  bar_style='info',
-                 orientation='horizontal') -> None:
+                 orientation='horizontal',
+                 tail_text: str = '') -> None:
         self.min_val = min_value
         self.max_val = max_values
         self.description = description
+        self.tail_text = tail_text
         self.progress_bar = widgets.FloatProgress(
             value=initial_value,
             min=min_value,
@@ -98,6 +101,17 @@ def __init__(self, initial_value: float = 0.0,
             bar_style=bar_style,
             orientation=orientation
         )
+        self.tail_label = widgets.Label(self.tail_text)
+        self.progress_widget = widgets.Box(
+            [
+            self.progress_bar,
+            self.tail_label
+            ]
+        )
+
+    def update_tail_text(self, new_text: str):
+        self.tail_text = new_text
+        self.tail_label.value = new_text
 
     def update(self, value: float, bar_status: str = 'info'):
         if value <= self.max_val and value >= self.min_val:
@@ -106,7 +120,7 @@ def update(self, value: float, bar_status: str = 'info'):
             self._respond_calculus()
 
     def display(self):
-        display(self.progress_bar)
+        display(self.progress_widget)
 
     def get_progress_bar(self) -> widgets.FloatProgress:
         return self.progress_bar
@@ -115,9 +129,10 @@ def reset_progress_bar(self):
         self.progress_bar.value = 0.0
         self.progress_bar.description = self.description
         self.progress_bar.bar_style = 'info'
+        self.tail_text = ''
 
     def hide(self):
-        self.progress_bar.layout.display = "none"
+        self.progress_widget.layout.display = "none"
 
     def _respond_calculus(self, default_description='success'):
         value = self.progress_bar.value
@@ -134,9 +149,8 @@ def execute_simulation(num_users: int,
                        df_recipes: pd.DataFrame,
                        progress_bar: FloatProgressBar = None,
                        num_simultaneous_allergies: int = 2) -> Tuple[Any, Any, Any]:
-    # Todo check dictionaries probabilities
-    # Todo: Get values from dictionaries and send to the simulation function
-    df_user_join, table, new_tracking = run_full_simulation(
+    # Execute simulation 
+    df_user_join, table, new_tracking, simulation_parameters = run_full_simulation(
         num_users=num_users,
         chose_dist=chose_dist,
         delta_dist_dict=values_from_dictionary(
@@ -169,7 +183,7 @@ def execute_simulation(num_users: int,
         num_days=num_days,
         multiple_allergies_number=num_simultaneous_allergies
     )
-    return df_user_join, table, new_tracking
+    return df_user_join, table, new_tracking, simulation_parameters
 
 
 def check_sum_proba(dict_proba, round_digits=1):
@@ -544,13 +558,14 @@ def execute_simulation(self):
             else:
                 print("simulation starting")
                 self.progress_bar.display()
-            # load recipes data todo make this parametrizable
+            # load recipes data
+            #TODO: make this parametrize
             current_dir = os.getcwd()
             default_path_recipes = 'recipes/recipes_sampling_1000.csv'
             # default_path_recipes = "recipes/extended_processed_recipes_dataset_id.csv"
             df_recipes = pd.read_csv(os.path.join(current_dir, default_path_recipes),
                                      sep="|", index_col=0)
-            df_user_join, table, new_tracking_df = execute_simulation(num_users=self.num_users.value,
+            df_user_join, table, new_tracking_df, sim_parameters = execute_simulation(num_users=self.num_users.value,
                                                                       chose_dist=self.delta_dist_chose,
                                                                       dictionaries=self.dictionaries,
                                                                       probability_transition_matrix=probability_transition_matrix,
@@ -562,8 +577,9 @@ def execute_simulation(self):
             self.table = table
             # save result to a temporal  directory
             base_output_path = os.path.join(os.getcwd(), "outputs")
-            folder_output_name = dt.datetime.now().strftime('%d-%m-%Y_%H-%M-%S')
+            folder_output_name = dt.datetime.now().strftime('%d-%m-%Y_%H-%M-%S') 
             files_dict = {
+                "simulation_parameters.npy": sim_parameters,
                 "users_dataset.csv": df_user_join,
                 "tracking.csv": new_tracking_df,
                 "summary_table.html": table.render(),
@@ -638,7 +654,19 @@ def execute_simulation(self):
 
     def button_callback(self, b):
         self.execute_simulation()
-
+
+def process_uploaded_file(uploaded_file):
+    content = io.StringIO(uploaded_file.decode('utf-8'))
+    df = pd.read_csv(content, sep="|", index_col=0)
+    print(df.head(3))
+    return df
+
+def on_file_change(change):
+    print(change)
+    uploaded_filename = next(iter(change.new))
+    print(f"File uploaded: {uploaded_filename}")
+    uploaded_file = change.new[uploaded_filename]['content']
+    process_uploaded_file(uploaded_file)
 
 def build_full_ui():
     # Get initializers
@@ -792,6 +820,26 @@ def build_full_ui():
     }
     dict_widgets['delta_distribution'] = {"widget_list": distribution_selector.get_output(),
                                           "titles": "Appreciation feedback (delta)"}
+    recipes_file_upload = widgets.FileUpload(
+        accept='.csv',
+        multiple=False,
+        description='Upload recipes:')
+    sep_char = '|'
+    # dict_widgets['upload_recipes'] = {"widget_list": widgets.VBox(
+    #     [
+    #     widgets.Label('Upload a recipe in a csv file and specify the separator.'),
+    #     widgets.Box([
+    #     recipes_file_upload,
+    #     widgets.Text(
+    #         value=sep_char,
+    #         placeholder='Introduce the csv separator',
+    #         description='Separator:',
+    #         disabled=False,
+    #         style={'description_width': 'initial'}
+    #     )
+    #         ])
+    #     ]),
+    #     "titles": "Upload recipes (Optional)"}
     # UI displaying
     style = {'description_width': 'initial'}
     NUM_USERS = widgets.IntText(
@@ -812,6 +860,7 @@ def build_full_ui():
                              names='value')
     allergies_preset_combo.observe(allergies_preset_event_handler.dropDownChange,
                                    names='value')
+    recipes_file_upload.observe(on_file_change, names='value')
     # BMI preset connection
     bmi_preset_event_handler = UpdateDropdown(
         value_dict=BMI_probabilities,