Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bugfix: database breakage during update #511

Merged
merged 6 commits into from
Oct 15, 2024
Merged

bugfix: database breakage during update #511

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bcd219c
Assets imported from beta cisTEM are now usable
twagner9 Jun 4, 2024
661c4dd
Filling in more columns
twagner9 Jun 4, 2024
68ae856
gui: added a progress dialog for schema update
twagner9 Aug 9, 2024
f80b4e8
Update comments, refactor some variables
twagner9 Oct 9, 2024
feb4b18
feat: added automatic backup option
twagner9 Oct 11, 2024
4aa4595
fix: variable names and comments, backup fail
twagner9 Oct 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 203 additions & 3 deletions src/core/database.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2362,17 +2362,217 @@ void Database::GetRefinementAngularDistributionHistogramData(long wanted_refinem
EndBatchSelect( );
}

bool Database::UpdateSchema(ColumnChanges columns) {
bool Database::UpdateSchema(ColumnChanges columns, UpdateProgressTracker* progress_bar, unsigned long total_num_rows, int normalized_increments) {
using namespace database_schema;
CreateAllTables( );
char format;
wxString column_format;
char format;
wxString column_format;
int col_counter;
unsigned long increments_processed = 0;
int current_progress = 0;
int previous_progress = 0;
bool should_update_text = false; // Modified in UpdateProgressTracker::OnUpdateProgress
bool output_pixel_size_added = false;

// Assitive lambda function used in helping to update the progress bar
auto calculate_current_percentage = [&increments_processed, &total_num_rows, &normalized_increments]( ) -> int {
double percent_completion = (double(increments_processed) / double(total_num_rows)) * normalized_increments;
return static_cast<int>(percent_completion);
};

for ( ColumnChange& column : columns ) {
format = std::get<COLUMN_CHANGE_TYPE>(column);
column_format = map_type_char_to_sqlite_string(format);
ExecuteSQL(wxString::Format("ALTER TABLE %s ADD COLUMN %s %s;", std::get<COLUMN_CHANGE_TABLE>(column), std::get<COLUMN_CHANGE_NAME>(column), column_format));

// checks for specific problem :-
// output pixel size was not added onto the end, and it needs to have a value set.

if ( std::get<COLUMN_CHANGE_NAME>(column) == "OUTPUT_PIXEL_SIZE" )
output_pixel_size_added = true;

if ( progress_bar ) {
increments_processed++; // Not actually updating rows, but altering tables; but, need one variable for tracking
current_progress = calculate_current_percentage( );

if ( current_progress > previous_progress ) {
progress_bar->OnUpdateProgress(current_progress, "Making column changes...", should_update_text);
previous_progress = current_progress;
}
}
}
should_update_text = true;

UpdateVersion( );

// do the more complicated post work..

if ( output_pixel_size_added ) {
// Grab IDs first, or else we can't properly update progress bar
wxArrayInt ref_pkg_ids = ReturnIntArrayFromSelectCommand("select REFINEMENT_PACKAGE_ASSET_ID from REFINEMENT_PACKAGE_ASSETS");
ExecuteSQL("drop table if exists cistem_schema_update_temp_table");
ExecuteSQL("alter table REFINEMENT_PACKAGE_ASSETS rename to cistem_schema_update_temp_table");
CreateAllTables( ); // should now be correct order but blank..

std::vector<wxString> all_columns;

for ( TableData& table : static_tables ) {

if ( std::get<TABLE_NAME>(table) == "REFINEMENT_PACKAGE_ASSETS" ) {
for ( col_counter = 0; col_counter < std::get<TABLE_COLUMNS>(table).size( ); col_counter++ ) {
all_columns.push_back(std::get<TABLE_COLUMNS>(table)[col_counter]);
}
}
}

wxString sql_command;
sql_command = "insert into REFINEMENT_PACKAGE_ASSETS select ";

for ( col_counter = 0; col_counter < all_columns.size( ); col_counter++ ) {
sql_command += all_columns[col_counter];

if ( col_counter < all_columns.size( ) - 1 )
sql_command += ", ";
else
sql_command += " from cistem_schema_update_temp_table;";
}

ExecuteSQL(sql_command);
ExecuteSQL("drop table if exists cistem_schema_update_temp_table");

// now we need to go and set this value to the output pixel size..
double current_pixel_size;
int num_particles = 0;

for ( int refinement_package_counter = 0; refinement_package_counter < ref_pkg_ids.GetCount( ); refinement_package_counter++ ) {
current_pixel_size = ReturnSingleDoubleFromSelectCommand(wxString::Format("select pixel_size from refinement_package_contained_particles_%i", ref_pkg_ids[refinement_package_counter]));
ExecuteSQL(wxString::Format("update refinement_package_assets set output_pixel_size = %f where refinement_package_asset_id = %i", current_pixel_size, ref_pkg_ids[refinement_package_counter]));

// Update loading bar
if ( progress_bar ) {
num_particles = ReturnSingleIntFromSelectCommand(wxString::Format("select COUNT(*) from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", ref_pkg_ids[refinement_package_counter]));
increments_processed += num_particles;
current_progress = calculate_current_percentage( );

if ( current_progress > previous_progress ) {
progress_bar->OnUpdateProgress(current_progress, "Updating refinement package(s) pixel size...", should_update_text);
previous_progress = current_progress;
}
}
}
should_update_text = true;
// Next, make sure pixel size, aberration, voltage, and amplitude contrast are being updated where needed
// This ensures updates from beta databases are successful
{
double classification_held_pixel_size = 0.0;
double refinement_held_pixel_size = 0.0;
double contained_particles_pixel_size = 0.0;
double aberration = 0.0;
double voltage = 0.0;
double amplitude_contrast = 0.0;
double defocus_1 = 0.0;
double defocus_2 = 0.0;
double defocus_angle = 0.0;
double phase_shift = 0.0;
int corresponding_refinement_package_id = 0;
int refinement_number_of_classes = 0;
wxArrayInt classification_ids = ReturnIntArrayFromSelectCommand(wxString::Format("select CLASSIFICATION_ID from CLASSIFICATION_LIST"));
wxArrayInt refinement_ids = ReturnIntArrayFromSelectCommand(wxString::Format("select REFINEMENT_ID from REFINEMENT_LIST"));

// Use the refinement_package_asset_id from each table to update the needed variables
// First do classification results
for ( int classification_result_counter = 0; classification_result_counter < classification_ids.GetCount( ); classification_result_counter++ ) {
corresponding_refinement_package_id = ReturnSingleIntFromSelectCommand(wxString::Format("select REFINEMENT_PACKAGE_ASSET_ID from CLASSIFICATION_LIST where CLASSIFICATION_ID = %i", classification_ids[classification_result_counter]));
classification_held_pixel_size = ReturnSingleDoubleFromSelectCommand(wxString::Format("select PIXEL_SIZE from CLASSIFICATION_RESULT_%i", classification_ids[classification_result_counter]));
contained_particles_pixel_size = ReturnSingleDoubleFromSelectCommand(wxString::Format("select PIXEL_SIZE from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));

// If pixel size doesn't match, other parameters probably don't -- fix classification tables; fill all NULL columns.
if ( contained_particles_pixel_size != classification_held_pixel_size ) {
aberration = ReturnSingleDoubleFromSelectCommand(wxString::Format("select SPHERICAL_ABERRATION from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));
voltage = ReturnSingleDoubleFromSelectCommand(wxString::Format("select MICROSCOPE_VOLTAGE from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));
amplitude_contrast = ReturnSingleDoubleFromSelectCommand(wxString::Format("select AMPLITUDE_CONTRAST from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));
defocus_1 = ReturnSingleDoubleFromSelectCommand(wxString::Format("select DEFOCUS_1 from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));
defocus_2 = ReturnSingleDoubleFromSelectCommand(wxString::Format("select DEFOCUS_2 from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));
defocus_angle = ReturnSingleDoubleFromSelectCommand(wxString::Format("select DEFOCUS_ANGLE from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));
phase_shift = ReturnSingleDoubleFromSelectCommand(wxString::Format("select PHASE_SHIFT from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));

ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set PIXEL_SIZE = %f", classification_ids[classification_result_counter], contained_particles_pixel_size));
ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set CS = %f", classification_ids[classification_result_counter], aberration));
ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set AMPLITUDE_CONTRAST = %f", classification_ids[classification_result_counter], amplitude_contrast));
ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set VOLTAGE = %f", classification_ids[classification_result_counter], voltage));
ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set DEFOCUS_1 = %f", classification_ids[classification_result_counter], defocus_1));
ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set DEFOCUS_2 = %f", classification_ids[classification_result_counter], defocus_2));
ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set DEFOCUS_ANGLE = %f", classification_ids[classification_result_counter], defocus_angle));
ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set PHASE_SHIFT = %f", classification_ids[classification_result_counter], phase_shift));

// Then fill in the columns that would remain 0.0 as the values didn't exist in the beta version
ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set BEAM_TILT_X = 0.0 where BEAM_TILT_X is null", classification_ids[classification_result_counter]));
ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set BEAM_TILT_Y = 0.0 where BEAM_TILT_Y is null", classification_ids[classification_result_counter]));
ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set IMAGE_SHIFT_X = 0.0 where IMAGE_SHIFT_X is null", classification_ids[classification_result_counter]));
ExecuteSQL(wxString::Format("update CLASSIFICATION_RESULT_%i set IMAGE_SHIFT_Y = 0.0 where IMAGE_SHIFT_Y is null", classification_ids[classification_result_counter]));
}

// Update the loading bar
if ( progress_bar ) {
num_particles = ReturnSingleIntFromSelectCommand(wxString::Format("select COUNT(*) from CLASSIFICATION_RESULT_%i", classification_ids[classification_result_counter]));
increments_processed += num_particles;
current_progress = calculate_current_percentage( );
if ( current_progress > previous_progress ) {
progress_bar->OnUpdateProgress(current_progress, "Updating classification result(s)...", should_update_text);
previous_progress = current_progress;
}
}
}
should_update_text = true;

// Now do refinement results; first loop over the refinement_ids, then loop over the number of classes.
for ( int refinement_result_counter = 0; refinement_result_counter < refinement_ids.GetCount( ); refinement_result_counter++ ) {
refinement_number_of_classes = ReturnSingleIntFromSelectCommand(wxString::Format("select NUMBER_OF_CLASSES from REFINEMENT_LIST where REFINEMENT_ID = %i", refinement_ids[refinement_result_counter])) + 1;
for ( int refinement_result_class_counter = 1; refinement_result_class_counter < refinement_number_of_classes; refinement_result_class_counter++ ) {
corresponding_refinement_package_id = ReturnSingleIntFromSelectCommand(wxString::Format("select REFINEMENT_PACKAGE_ASSET_ID from REFINEMENT_LIST where REFINEMENT_ID = %i", refinement_ids[refinement_result_counter]));
refinement_held_pixel_size = ReturnSingleDoubleFromSelectCommand(wxString::Format("select PIXEL_SIZE from REFINEMENT_RESULT_%i_%i", refinement_ids[refinement_result_counter], refinement_result_class_counter));
contained_particles_pixel_size = ReturnSingleDoubleFromSelectCommand(wxString::Format("select PIXEL_SIZE from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));

// If pixel size doesn't match, other parameters probably don't; repeat above process for refinement result tables
if ( contained_particles_pixel_size != classification_held_pixel_size ) {
aberration = ReturnSingleDoubleFromSelectCommand(wxString::Format("select SPHERICAL_ABERRATION from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));
voltage = ReturnSingleDoubleFromSelectCommand(wxString::Format("select MICROSCOPE_VOLTAGE from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));
amplitude_contrast = ReturnSingleDoubleFromSelectCommand(wxString::Format("select AMPLITUDE_CONTRAST from REFINEMENT_PACKAGE_CONTAINED_PARTICLES_%i", corresponding_refinement_package_id));

ExecuteSQL(wxString::Format("update REFINEMENT_RESULT_%i_%i set PIXEL_SIZE = %f", refinement_ids[refinement_result_counter], refinement_result_class_counter, contained_particles_pixel_size));
ExecuteSQL(wxString::Format("update REFINEMENT_RESULT_%i_%i set MICROSCOPE_CS = %f", refinement_ids[refinement_result_counter], refinement_result_class_counter, aberration));
ExecuteSQL(wxString::Format("update REFINEMENT_RESULT_%i_%i set AMPLITUDE_CONTRAST = %f", refinement_ids[refinement_result_counter], refinement_result_class_counter, amplitude_contrast));
ExecuteSQL(wxString::Format("update REFINEMENT_RESULT_%i_%i set MICROSCOPE_VOLTAGE = %f", refinement_ids[refinement_result_counter], refinement_result_class_counter, voltage));

// Then fill in the columns that would remain 0.0 as the values didn't exist in the beta version
ExecuteSQL(wxString::Format("update REFINEMENT_RESULT_%i_%i set BEAM_TILT_X = 0.0 where BEAM_TILT_X is null", refinement_ids[refinement_result_counter], refinement_result_class_counter));
ExecuteSQL(wxString::Format("update REFINEMENT_RESULT_%i_%i set BEAM_TILT_Y = 0.0 where BEAM_TILT_Y is null", refinement_ids[refinement_result_counter], refinement_result_class_counter));
ExecuteSQL(wxString::Format("update REFINEMENT_RESULT_%i_%i set IMAGE_SHIFT_X = 0.0 where IMAGE_SHIFT_X is null", refinement_ids[refinement_result_counter], refinement_result_class_counter));
ExecuteSQL(wxString::Format("update REFINEMENT_RESULT_%i_%i set IMAGE_SHIFT_Y = 0.0 where IMAGE_SHIFT_Y is null", refinement_ids[refinement_result_counter], refinement_result_class_counter));
}

// Update the loading bar
if ( progress_bar ) {
num_particles = ReturnSingleIntFromSelectCommand(wxString::Format("select COUNT(*) from REFINEMENT_RESULT_%i_%i", refinement_ids[refinement_result_counter], refinement_result_class_counter));
increments_processed += num_particles;
current_progress = calculate_current_percentage( );

if ( current_progress > previous_progress ) {
progress_bar->OnUpdateProgress(current_progress, "Updating refinement result(s)...", should_update_text);
previous_progress = current_progress;
}
}
}
}
}

// Now update the last 3 columns of the estimated ctf parameters table
{
ExecuteSQL(wxString::Format("update ESTIMATED_CTF_PARAMETERS set ICINESS = 0.0 where ICINESS is null"));
ExecuteSQL(wxString::Format("update ESTIMATED_CTF_PARAMETERS set TILT_ANGLE = 0.0 where TILT_ANGLE is null"));
ExecuteSQL(wxString::Format("update ESTIMATED_CTF_PARAMETERS set TILT_AXIS = 0.0 where TILT_AXIS is null"));
}
}
return true;
}

Expand Down
3 changes: 2 additions & 1 deletion src/core/database.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include "../constants/constants.h"
#include "../gui/UpdateProgressTracker.h"

class Database {

Expand Down Expand Up @@ -371,7 +372,7 @@ class Database {

using ColumnChanges = std::vector<ColumnChange>;
std::pair<TableChanges, ColumnChanges> CheckSchema( );
bool UpdateSchema(ColumnChanges columns);
bool UpdateSchema(ColumnChanges columns, UpdateProgressTracker* progress_bar = nullptr, unsigned long total_num_rows = 0, int normalized_increments = 100);
bool UpdateVersion( );
};

Expand Down
1 change: 0 additions & 1 deletion src/core/database_schema.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ std::vector<TableData> dynamic_tables{
{"RUN_PROFILE_COMMANDS_", "ptiiiii", {"COMMANDS_NUMBER", "COMMAND_STRING", "NUMBER_OF_COPIES", "NUMBER_OF_THREADS_PER_COPY", "OVERRIDE_TOTAL_NUMBER_OF_COPIES", "OVERIDDEN_TOTAL_NUMBER_OF_COPIES", "DELAY_TIME_IN_MS"}},
{"MOVIE_IMPORT_DEFAULTS", "prrrrititirirrriii", {"NUMBER", "VOLTAGE", "SPHERICAL_ABERRATION", "PIXEL_SIZE", "EXPOSURE_PER_FRAME", "MOVIES_ARE_GAIN_CORRECTED", "GAIN_REFERENCE_FILENAME", "MOVIES_ARE_DARK_CORRECTED", "DARK_REFERENCE_FILENAME", "RESAMPLE_MOVIES", "DESIRED_PIXEL_SIZE", "CORRECT_MAG_DISTORTION", "MAG_DISTORTION_ANGLE", "MAG_DISTORTION_MAJOR_SCALE", "MAG_DISTORTION_MINOR_SCALE", "PROTEIN_IS_WHITE", "EER_SUPER_RES_FACTOR", "EER_FRAMES_PER_IMAGE"}},
{"IMAGE_IMPORT_DEFAULTS", "prrri", {"NUMBER", "VOLTAGE", "SPHERICAL_ABERRATION", "PIXEL_SIZE", "PROTEIN_IS_WHITE"}},
{"REFINEMENT_PACKAGE_CONTAINED_PARTICLES_", "piirrrrrrrrrri", {"ORIGINAL_PARTICLE_POSITION_ASSET_ID", "PARENT_IMAGE_ASSET_ID", "POSITION_IN_STACK", "X_POSITION", "Y_POSITION", "PIXEL_SIZE", "DEFOCUS_1", "DEFOCUS_2", "DEFOCUS_ANGLE", "PHASE_SHIFT", "SPHERICAL_ABERRATION", "MICROSCOPE_VOLTAGE", "AMPLITUDE_CONTRAST", "ASSIGNED_SUBSET"}},
{"STARTUP_RESULT_", "pl", {"CLASS_NUMBER", "VOLUME_ASSET_ID"}},
{"CLASSIFICATION_SELECTION_", "pl", {"SELECTION_NUMBER", "CLASS_AVERAGE_NUMBER"}},
{"CLASSIFICATION_RESULT_", "Prrrirrrrrrrrrrrrrr", {"POSITION_IN_STACK", "PSI", "XSHIFT", "YSHIFT", "BEST_CLASS", "SIGMA", "LOGP", "PIXEL_SIZE", "VOLTAGE", "CS", "AMPLITUDE_CONTRAST", "DEFOCUS_1", "DEFOCUS_2", "DEFOCUS_ANGLE", "PHASE_SHIFT", "BEAM_TILT_X", "BEAM_TILT_Y", "IMAGE_SHIFT_X", "IMAGE_SHIFT_Y"}},
Expand Down
Loading
Loading