Merge branch 'master' of https://github.com/madgik/exareme into Multi…

…pleHistogramBugFix
madgik · Oct 16, 2020 · 0ab4011 · 0ab4011
2 parents 5a4a8c4 + c47293d
commit 0ab4011
Showing 1 changed file with 14 additions and 4 deletions.
diff --git a/Exareme-Docker/files/root/exareme/convert-csv-dataset-to-db.py b/Exareme-Docker/files/root/exareme/convert-csv-dataset-to-db.py
@@ -190,32 +190,41 @@ def addCSVInTheDataTable(csvFilePath, metadataDictionary, cur):
         columnsString += ', ' + column
     columnsSectionOfSQLQuery = 'INSERT INTO DATA (' + columnsString + ') VALUES '
 
-
     # Insert data
-    number_of_rows = 0
+    numberOfRows = 0
     valuesSectionOfSQLQuery = '('
     for row in csvReader:
-        number_of_rows += 1
+        numberOfRows += 1
         for (value, column) in zip(row, csvHeader):
             if metadataDictionary[column] == 'text':
                 valuesSectionOfSQLQuery += "'" + value + "', "
             elif value == '':
                 valuesSectionOfSQLQuery += 'null, '
             else:
                 valuesSectionOfSQLQuery += value + ", "
-        if (number_of_rows % int(MAX_ROWS_TO_INSERT_INTO_SQL) == 0 or next(csvReader, None) == None):
+        if numberOfRows % int(MAX_ROWS_TO_INSERT_INTO_SQL) == 0:
             valuesSectionOfSQLQuery = valuesSectionOfSQLQuery[:-2]
             valuesSectionOfSQLQuery += ');'
 
             try:
                 cur.execute(columnsSectionOfSQLQuery + valuesSectionOfSQLQuery)
             except:
                 findErrorOnBulkInsertQuery(cur, valuesSectionOfSQLQuery, csvHeader, metadataDictionary, csvFilePath)
+                raise ValueError("Error inserting the CSV to the database.")
             valuesSectionOfSQLQuery = '('
         else:
             valuesSectionOfSQLQuery = valuesSectionOfSQLQuery[:-2]
             valuesSectionOfSQLQuery += '),('
 
+    if numberOfRows % int(MAX_ROWS_TO_INSERT_INTO_SQL) != 0:
+        valuesSectionOfSQLQuery = valuesSectionOfSQLQuery[:-3]
+        valuesSectionOfSQLQuery += ');'
+
+        try:
+            cur.execute(columnsSectionOfSQLQuery + valuesSectionOfSQLQuery)
+        except:
+            findErrorOnBulkInsertQuery(cur, valuesSectionOfSQLQuery, csvHeader, metadataDictionary, csvFilePath)
+
 
 def findErrorOnBulkInsertQuery(cur, valuesOfQuery, csvHeader, metadataDictionary, csvFilePath):
     # Removing the first and last parenthesis
@@ -226,6 +235,7 @@ def findErrorOnBulkInsertQuery(cur, valuesOfQuery, csvHeader, metadataDictionary
     for row in valuesOfQuery.split('),('):
         findErrorOnSqlQuery(cur, row.split(','), csvHeader, metadataDictionary, csvFilePath)
 
+
 def findErrorOnSqlQuery(cur, row, csvHeader, metadataDictionary, csvFilePath):
     # Insert the code column into the database and then update it for each row to find where the problem is
     firstRow = True