Merge pull request #511 from timothygrant80/output_pixel_size_db_upda…

…te_fix

bugfix: database breakage during update

src/Makefile.am

@@ -72,6 +72,7 @@ noinst_HEADERS = core/stopwatch.h \
                  core/eer_file.h \
                  gui/job_panel.h \
                  gui/gui_functions.h \
+                 gui/DatabaseUpdateDialog.h \
                  gui/ResultsDataViewListCtrl.h \
                  gui/BitmapPanel.h \
                  gui/PickingBitmapPanel.h \
@@ -638,6 +639,7 @@ cisTEM_SOURCES = programs/projectx/projectx.cpp \
                 gui/ProjectX_gui_settings.cpp \
                 gui/ProjectX_gui_unblur.cpp \
                 gui/ProjectX_gui_wizards.cpp \
+                gui/DatabaseUpdateDialog.cpp \
                 gui/ResultsDataViewListCtrl.cpp \
                 gui/MyMovieFilterDialog.cpp \
                 gui/MainFrame.cpp \

src/core/database.h

@@ -1,4 +1,5 @@
 #include "../constants/constants.h"
+#include "../gui/UpdateProgressTracker.h"
 
 class Database {
 
@@ -25,6 +26,7 @@ class Database {
 
     bool CreateNewDatabase(wxFileName database_file);
     bool Open(wxFileName file_to_open, bool disable_locking = false);
+    bool CopyDatabaseFile(wxFileName backup_db);
 
     inline void Begin( ) {
         if ( number_of_active_transactions == 0 )
@@ -371,7 +373,7 @@ class Database {
 
     using ColumnChanges = std::vector<ColumnChange>;
     std::pair<TableChanges, ColumnChanges> CheckSchema( );
-    bool                                   UpdateSchema(ColumnChanges columns);
+    bool                                   UpdateSchema(ColumnChanges columns, UpdateProgressTracker* progress_bar = nullptr, unsigned long total_num_rows = 0, int normalized_increments = 100);
     bool                                   UpdateVersion( );
 };
 

src/core/database_schema.h

@@ -57,7 +57,6 @@ std::vector<TableData> dynamic_tables{
         {"RUN_PROFILE_COMMANDS_", "ptiiiii", {"COMMANDS_NUMBER", "COMMAND_STRING", "NUMBER_OF_COPIES", "NUMBER_OF_THREADS_PER_COPY", "OVERRIDE_TOTAL_NUMBER_OF_COPIES", "OVERIDDEN_TOTAL_NUMBER_OF_COPIES", "DELAY_TIME_IN_MS"}},
         {"MOVIE_IMPORT_DEFAULTS", "prrrrititirirrriii", {"NUMBER", "VOLTAGE", "SPHERICAL_ABERRATION", "PIXEL_SIZE", "EXPOSURE_PER_FRAME", "MOVIES_ARE_GAIN_CORRECTED", "GAIN_REFERENCE_FILENAME", "MOVIES_ARE_DARK_CORRECTED", "DARK_REFERENCE_FILENAME", "RESAMPLE_MOVIES", "DESIRED_PIXEL_SIZE", "CORRECT_MAG_DISTORTION", "MAG_DISTORTION_ANGLE", "MAG_DISTORTION_MAJOR_SCALE", "MAG_DISTORTION_MINOR_SCALE", "PROTEIN_IS_WHITE", "EER_SUPER_RES_FACTOR", "EER_FRAMES_PER_IMAGE"}},
         {"IMAGE_IMPORT_DEFAULTS", "prrri", {"NUMBER", "VOLTAGE", "SPHERICAL_ABERRATION", "PIXEL_SIZE", "PROTEIN_IS_WHITE"}},
-        {"REFINEMENT_PACKAGE_CONTAINED_PARTICLES_", "piirrrrrrrrrri", {"ORIGINAL_PARTICLE_POSITION_ASSET_ID", "PARENT_IMAGE_ASSET_ID", "POSITION_IN_STACK", "X_POSITION", "Y_POSITION", "PIXEL_SIZE", "DEFOCUS_1", "DEFOCUS_2", "DEFOCUS_ANGLE", "PHASE_SHIFT", "SPHERICAL_ABERRATION", "MICROSCOPE_VOLTAGE", "AMPLITUDE_CONTRAST", "ASSIGNED_SUBSET"}},
         {"STARTUP_RESULT_", "pl", {"CLASS_NUMBER", "VOLUME_ASSET_ID"}},
         {"CLASSIFICATION_SELECTION_", "pl", {"SELECTION_NUMBER", "CLASS_AVERAGE_NUMBER"}},
         {"CLASSIFICATION_RESULT_", "Prrrirrrrrrrrrrrrrr", {"POSITION_IN_STACK", "PSI", "XSHIFT", "YSHIFT", "BEST_CLASS", "SIGMA", "LOGP", "PIXEL_SIZE", "VOLTAGE", "CS", "AMPLITUDE_CONTRAST", "DEFOCUS_1", "DEFOCUS_2", "DEFOCUS_ANGLE", "PHASE_SHIFT", "BEAM_TILT_X", "BEAM_TILT_Y", "IMAGE_SHIFT_X", "IMAGE_SHIFT_Y"}},

src/gui/DatabaseUpdateDialog.cpp

@@ -0,0 +1,29 @@
+#include "DatabaseUpdateDialog.h"
+
+DatabaseUpdateDialog::DatabaseUpdateDialog(wxWindow* parent, wxString db_changes) : DatabaseUpdateDialogParent(parent) {
+    schema_changes = db_changes;
+    SchemaChangesTextCtrl->AppendText(db_changes);
+}
+
+void DatabaseUpdateDialog::OnButtonClicked(wxCommandEvent& event) {
+    int button_id = event.GetId( );
+
+    enum UpdateOptions { CANCEL            = 1,
+                         UPDATE_ONLY       = 2,
+                         BACKUP_AND_UPDATE = 3 };
+
+    switch ( button_id ) {
+        case wxID_CANCEL:
+            EndModal(CANCEL);
+            break;
+        case wxID_UPDATE_ONLY:
+            EndModal(UPDATE_ONLY);
+            break;
+        case wxID_BACKUP_AND_UPDATE:
+            EndModal(BACKUP_AND_UPDATE);
+            break;
+        default:
+            EndModal(CANCEL);
+            break;
+    }
+}

src/gui/DatabaseUpdateDialog.h

@@ -0,0 +1,16 @@
+#ifndef _SRC_GUI_DATABASE_UDPATE_DIALOG_H_
+#define _SRC_GUI_DATABASE_UPDATE_DIALOG_H_
+
+#include "ProjectX_gui_main.h"
+
+class DatabaseUpdateDialog : public DatabaseUpdateDialogParent {
+  public:
+    // DatabaseUpdateDialog(wxWindow* parent);
+    DatabaseUpdateDialog(wxWindow* parent, wxString db_changes);
+
+    void OnButtonClicked(wxCommandEvent& event);
+
+  private:
+    wxString schema_changes;
+};
+#endif

src/gui/MainFrame.cpp

@@ -1,5 +1,6 @@
 //#include "../core/core_headers.h"
 #include "../core/gui_core_headers.h"
+#include "DatabaseUpdateDialog.h"
 #include <wx/richmsgdlg.h>
 
 #define SERVER_ID 100
@@ -532,38 +533,72 @@ void MyMainFrame::OpenProject(wxString project_filename) {
             current_project.Close(false, false);
             return;
         }
+        // We need to see if we want to update
         else if ( current_project.integer_database_version != INTEGER_DATABASE_VERSION || current_project.cistem_version_text != CISTEM_VERSION_TEXT ) {
             auto     schema_comparison = current_project.database.CheckSchema( );
-            wxString message;
-            wxString button;
-            wxString changes = "";
+            wxString changes           = "";
             if ( schema_comparison.first.size( ) == 0 && schema_comparison.second.size( ) == 0 ) {
-                message = "However, there seem to be no changes in the file format\n\nAttempt to open the project?";
-                button  = "Open";
+                wxRichMessageDialog* my_dialog = new wxRichMessageDialog(this, wxString::Format("This project was last opened by a different cisTEM version:-\n\nCurrent Version: \t %s\nProject Version: \t %s\n\nHowever, there seem to be no changes to the file format.\n\nAttempt to open the project?,", CISTEM_VERSION_TEXT, current_project.cistem_version_text), "Database from different cisTEM version?", wxICON_ERROR | wxYES_NO | wxNO_DEFAULT);
+                if ( my_dialog->ShowModal( ) == wxID_YES )
+                    my_dialog->Destroy( );
+                else {
+                    my_dialog->Destroy( );
+                    current_project.Close(false, false);
+                    return;
+                }
             }
+            // database schemas weren't the same; we're going to have to do a manual update
             else {
-                message = "cisTEM can try to update the format. It is wise to make a backup of the database before trying this.\n\nAttempt to update the project?";
-                button  = "Update";
+                // Logic for getting table differences
                 for ( auto& table : schema_comparison.first ) {
                     changes += wxString::Format("Add Table: \t %s\n", table);
                 }
                 for ( auto& column : schema_comparison.second ) {
                     changes += wxString::Format("In Table: \t %s \tadd column: \t %s\n", std::get<0>(column), std::get<1>(column));
                 }
-            }
-            wxRichMessageDialog* my_dialog = new wxRichMessageDialog(this, wxString::Format("This project was last opened by a different cisTEM version :-\n\nCurrent Version: \t %s\nProject Version: \t %s\n\n%s", CISTEM_VERSION_TEXT, current_project.cistem_version_text, message), "Database from different cisTEM version?", wxICON_ERROR | wxYES_NO | wxNO_DEFAULT);
-            my_dialog->SetYesNoLabels(button, "Close");
-            if ( changes != wxString("") ) {
-                my_dialog->ShowDetailedText(changes);
-            }
-            if ( my_dialog->ShowModal( ) == wxID_YES ) {
-                my_dialog->Destroy( );
-                current_project.database.UpdateSchema(schema_comparison.second);
-            }
-            else {
-                my_dialog->Destroy( );
-                current_project.Close(false, false);
-                return;
+
+                int user_update_type = 0;
+
+                DatabaseUpdateDialog* update_dialog = new DatabaseUpdateDialog(this, changes);
+                // Show dialog and get user choice
+                user_update_type = update_dialog->ShowModal( );
+
+                // Create enum to avoid 'magic numbers' -- easier to identify at a glance what is happening in each case this way
+                enum UpdateOptions { CANCEL            = 1,
+                                     UPDATE_ONLY       = 2,
+                                     BACKUP_AND_UPDATE = 3 };
+
+                switch ( user_update_type ) {
+                    case CANCEL: {
+                        current_project.Close(false, false);
+                        return;
+                    }
+                    case UPDATE_ONLY: {
+                        UpdateDatabase(schema_comparison);
+                        break;
+                    }
+                    case BACKUP_AND_UPDATE: {
+                        // Create backup filename which simply appends _backup
+                        std::string tmp_backup_filename = std::string(current_project.database.ReturnFilename( ));
+                        tmp_backup_filename.insert(tmp_backup_filename.size( ) - 3, "_backup");
+                        wxFileName backup_db_filename(tmp_backup_filename);
+
+                        // Backup and update
+                        bool backup_succeeded = current_project.database.CopyDatabaseFile(backup_db_filename);
+                        if ( ! backup_succeeded ) {
+                            wxPrintf("Failed to backup database properly. Abandoning update attempt.\n");
+                            current_project.Close(false, false);
+                            return;
+                        }
+                        UpdateDatabase(schema_comparison);
+                        break;
+                    }
+                    default: {
+                        // Shouldn't really get here, so something went wrong, or user closed popup; do same as CANCEL case for safety
+                        current_project.Close(false, false);
+                        return;
+                    }
+                }
             }
         }
 
@@ -625,7 +660,7 @@ void MyMainFrame::OpenProject(wxString project_filename) {
         my_dialog->Update(7, "Opening project (loading CTF estimation results...)");
         //	current_project.database.AddCTFIcinessColumnIfNecessary();
         ctf_results_panel->FillBasedOnSelectCommand("SELECT DISTINCT IMAGE_ASSET_ID FROM ESTIMATED_CTF_PARAMETERS");
-        //	current_project.database.AddCTFIcinessColumnIfNecessary();
+//	current_project.database.AddCTFIcinessColumnIfNecessary();
 #ifdef EXPERIMENTAL
         my_dialog->Update(8, "Opening project (loading Match Template Results...)");
         match_template_results_panel->FillBasedOnSelectCommand("SELECT DISTINCT IMAGE_ASSET_ID FROM TEMPLATE_MATCH_LIST");
@@ -946,3 +981,83 @@ void MyMainFrame::SetTemplateMatchingWorkflow(bool triggered_by_gui_event) {
 void MyMainFrame::OnTemplateMatchingWorkflow(wxCommandEvent& event) {
     SetTemplateMatchingWorkflow(true);
 }
+
+void MyMainFrame::UpdateDatabase(std::pair<Database::TableChanges, Database::ColumnChanges>& schema_comparison) {
+    // 1. Estimate number of rows that will be updated
+    // Total number of particles can be very large; on the order of billions.
+    // Use long to ensure enough space.
+    long row_updates = 0;
+    int  normalized_increments; // This will adjust how long it takes for an update based on the total number of particles that will be processed
+
+    // Limit scope of temporarily needed arrays while estimating total number of increments...
+    {
+        wxArrayInt ref_pkg_ids = current_project.database.ReturnIntArrayFromSelectCommand("select REFINEMENT_PACKAGE_ASSET_ID from REFINEMENT_PACKAGE_ASSETS");
+        for ( int cur_pkg_id : ref_pkg_ids ) {
+            row_updates += current_project.database.ReturnSingleIntFromSelectCommand(wxString::Format("select COUNT(*) from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", cur_pkg_id));
+        }
+        wxArrayString refinement_result_tables = current_project.database.ReturnStringArrayFromSelectCommand("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'REFINEMENT_RESULT_%_%'");
+        for ( wxString table_name : refinement_result_tables ) {
+            row_updates += current_project.database.ReturnSingleIntFromSelectCommand(wxString::Format("SELECT COUNT(*) FROM '%s'", table_name));
+        }
+        wxArrayString classification_result_tables = current_project.database.ReturnStringArrayFromSelectCommand("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'CLASSIFICATION_RESULT_%'");
+        for ( wxString table_name : classification_result_tables ) {
+            row_updates += current_project.database.ReturnSingleIntFromSelectCommand(wxString::Format("SELECT COUNT(*) FROM '%s'", table_name));
+        }
+    } // End total increments estimation
+
+    row_updates += schema_comparison.second.size( );
+
+    // Adjusting the normalized increments based on total increments helps prevent GUI from freezing up and gives more accurate
+    // estimated time remaining. This may need additional fine tuning.
+    {
+        long tmp_row_updates = row_updates;
+        int  place_value     = 1;
+
+        // Get highest place value
+        while ( tmp_row_updates >= 10 ) {
+            tmp_row_updates /= 10;
+            place_value *= 10;
+        }
+
+        // Determine how many progress bar increments there should be based on largest place value
+        if ( place_value <= 100000 ) {
+            normalized_increments = 100;
+        }
+        else if ( place_value > 100000 && place_value < 10000000 ) {
+            normalized_increments = 1000;
+        }
+        // 10 million or greater
+        else {
+            normalized_increments = 10000;
+        }
+    } // End progress normalization
+
+    // 2. Instantiate progress dialog member variable so it can receive updates
+    update_progress_dialog = new OneSecondProgressDialog("Updating Database...", "Starting database update...", normalized_increments, this, wxPD_APP_MODAL | wxPD_REMAINING_TIME | wxPD_AUTO_HIDE | wxPD_SMOOTH);
+
+    // 3. Update, passing MainFrame's UpdateProgressTracker variables
+    current_project.database.UpdateSchema(schema_comparison.second, this, row_updates, normalized_increments);
+    OnCompletion( );
+}
+
+////////////////////////////////////
+/// TRACK DATABASE SCHEMA UPDATE
+////////////////////////////////////
+void MyMainFrame::OnUpdateProgress(int progress, wxString new_msg, bool& should_update_text) {
+    if ( ! should_update_text ) {
+        update_progress_dialog->Update(progress);
+    }
+    else {
+        update_progress_dialog->Update(progress, new_msg);
+        should_update_text = false;
+    }
+}
+
+void MyMainFrame::OnCompletion( ) {
+    if ( update_progress_dialog ) {
+        update_progress_dialog->Destroy( );
+        update_progress_dialog = nullptr;
+    }
+}
+
+////////////////////////////////////

src/gui/MainFrame.h

@@ -1,8 +1,9 @@
 #ifndef _gui_MainFrame_h_
 #define _gui_MainFrame_h_
+#include "UpdateProgressTracker.h"
 
 /** Implementing MainFrame */
-class MyMainFrame : public MainFrame, public SocketCommunicator {
+class MyMainFrame : public MainFrame, public SocketCommunicator, public UpdateProgressTracker {
     bool                   is_fullscreen;
     cistem::workflow::Enum current_workflow;
     cistem::workflow::Enum previous_workflow;
@@ -28,6 +29,10 @@ class MyMainFrame : public MainFrame, public SocketCommunicator {
 
     virtual wxString ReturnName( ) { return "MainFrame"; }
 
+    // For schema update only; could be more generalized/re-used when needed; overrides from UpdateProgressTracker
+    void OnUpdateProgress(int progress, wxString new_msg, bool& should_update_text) override;
+    void OnCompletion( ) override;
+
     void RecalculateAssetBrowser(void);
     void OnCollapseAll(wxCommandEvent& event);
     void OnMenuBookChange(wxBookCtrlEvent& event);
@@ -119,6 +124,11 @@ class MyMainFrame : public MainFrame, public SocketCommunicator {
     }
 
     //LaunchJob(JobPanel *parent_panel, )
+
+  private:
+    // Only used in schema update at this point
+    OneSecondProgressDialog* update_progress_dialog;
+    void                     UpdateDatabase(std::pair<Database::TableChanges, Database::ColumnChanges>& schema_comparison);
 };
 
 #endif // _gui_MainFrame_h_