From 31608a184c97c1832ea9528281d481ec3989d9b1 Mon Sep 17 00:00:00 2001 From: Peter Petrik Date: Thu, 5 Oct 2023 15:32:04 +0200 Subject: [PATCH] Speed improvements of home page / sync preparation speed (#2820) * fetch only info about 50 projects in listProjectsByNameApi * cache calculation of local files checksums * do not resave cache file if there are no modifications * merge ProjectStatuses to NeedsSync * first check if we need sync because of server version, after that because of local changes * create optimized compare function --- app/CMakeLists.txt | 2 + app/projectsmodel.cpp | 3 + app/qml/components/ProjectDelegateItem.qml | 6 +- app/test/inputtests.cpp | 6 + app/test/testmerginapi.cpp | 35 ++++-- app/test/testprojectchecksumcache.cpp | 138 +++++++++++++++++++++ app/test/testprojectchecksumcache.h | 29 +++++ core/CMakeLists.txt | 2 + core/coreutils.cpp | 56 ++++----- core/coreutils.h | 11 +- core/merginapi.cpp | 127 +++++++++++++++---- core/merginapi.h | 20 ++- core/project.cpp | 32 ++--- core/project.h | 4 +- core/projectchecksumcache.cpp | 106 ++++++++++++++++ core/projectchecksumcache.h | 62 +++++++++ test/CMakeLists.txt | 1 + 17 files changed, 536 insertions(+), 104 deletions(-) create mode 100644 app/test/testprojectchecksumcache.cpp create mode 100644 app/test/testprojectchecksumcache.h create mode 100644 core/projectchecksumcache.cpp create mode 100644 core/projectchecksumcache.h diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index fc9a1a0fe..b13513dbd 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -195,6 +195,7 @@ if (ENABLE_TESTS) test/testutilsfunctions.cpp test/testvariablesmanager.cpp test/testactiveproject.cpp + test/testprojectchecksumcache.cpp ) set(MM_HDRS @@ -220,6 +221,7 @@ if (ENABLE_TESTS) test/testutilsfunctions.h test/testvariablesmanager.h test/testactiveproject.h + test/testprojectchecksumcache.h ) endif () diff --git a/app/projectsmodel.cpp b/app/projectsmodel.cpp index ed52e7523..c53c18caf 100644 --- a/app/projectsmodel.cpp +++ b/app/projectsmodel.cpp @@ -268,6 +268,9 @@ void ProjectsModel::mergeProjects( const MerginProjectsList &merginProjects, Mer else if ( project.local.hasMerginMetadata() ) { // App is starting - loads all local projects from a device + // OR + // We do not have server info because the project was ignored + // (listProjectsByName API limits response to max 50 projects) project.mergin.projectName = project.local.projectName; project.mergin.projectNamespace = project.local.projectNamespace; project.mergin.status = ProjectStatus::projectStatus( project ); diff --git a/app/qml/components/ProjectDelegateItem.qml b/app/qml/components/ProjectDelegateItem.qml index 43d007179..a350903ff 100644 --- a/app/qml/components/ProjectDelegateItem.qml +++ b/app/qml/components/ProjectDelegateItem.qml @@ -57,8 +57,7 @@ Rectangle { if ( projectIsLocal && projectIsMergin ) { // downloaded mergin projects - if ( projectStatus === ProjectStatus.OutOfDate || - projectStatus === ProjectStatus.Modified ) { + if ( projectStatus === ProjectStatus.NeedsSync ) { return InputStyle.syncIcon } return "" // no icon if this project does not have changes @@ -76,8 +75,7 @@ Rectangle { function getMoreMenuItems() { if ( projectIsMergin && projectIsLocal ) { - if ( ( projectStatus === ProjectStatus.OutOfDate || - projectStatus === ProjectStatus.Modified ) ) + if ( ( projectStatus === ProjectStatus.NeedsSync ) ) return "sync,changes,remove" return "changes,remove" diff --git a/app/test/inputtests.cpp b/app/test/inputtests.cpp index 4093f8a69..16bad90aa 100644 --- a/app/test/inputtests.cpp +++ b/app/test/inputtests.cpp @@ -30,6 +30,7 @@ #include "test/testmaptools.h" #include "test/testlayertree.h" #include "test/testactiveproject.h" +#include "test/testprojectchecksumcache.h" #if not defined APPLE_PURCHASING #include "test/testpurchasing.h" @@ -181,6 +182,11 @@ int InputTests::runTest() const TestActiveProject activeProjectTest( mApi ); nFailed = QTest::qExec( &activeProjectTest, mTestArgs ); } + else if ( mTestRequested == "--testProjectChecksumCache" ) + { + TestProjectChecksumCache projectChecksumTest; + nFailed = QTest::qExec( &projectChecksumTest, mTestArgs ); + } #if not defined APPLE_PURCHASING else if ( mTestRequested == "--testPurchasing" ) { diff --git a/app/test/testmerginapi.cpp b/app/test/testmerginapi.cpp index 98f23e014..232ee27c6 100644 --- a/app/test/testmerginapi.cpp +++ b/app/test/testmerginapi.cpp @@ -1,6 +1,7 @@ #include #include +#include "projectchecksumcache.h" #include "testmerginapi.h" #include "inpututils.h" #include "coreutils.h" @@ -533,7 +534,7 @@ void TestMerginApi::testMultiChunkUploadDownload() bigFile.write( QByteArray( 1024 * 1024, static_cast( 'A' + i ) ) ); // AAAA.....BBBB.....CCCC..... bigFile.close(); - QByteArray checksum = MerginApi::getChecksum( bigFilePath ); + QByteArray checksum = CoreUtils::calculateChecksum( bigFilePath ); QVERIFY( !checksum.isEmpty() ); // upload @@ -545,7 +546,7 @@ void TestMerginApi::testMultiChunkUploadDownload() downloadRemoteProject( mApi, mUsername, projectName ); // verify it's there and with correct content - QByteArray checksum2 = MerginApi::getChecksum( bigFilePath ); + QByteArray checksum2 = CoreUtils::calculateChecksum( bigFilePath ); QVERIFY( QFileInfo::exists( bigFilePath ) ); QCOMPARE( checksum, checksum2 ); } @@ -566,7 +567,7 @@ void TestMerginApi::testEmptyFileUploadDownload() QFile::copy( mTestDataPath + "/" + TEST_EMPTY_FILE_NAME, emptyFileDestinationPath ); QVERIFY( QFileInfo::exists( emptyFileDestinationPath ) ); - QByteArray checksum = MerginApi::getChecksum( emptyFileDestinationPath ); + QByteArray checksum = CoreUtils::calculateChecksum( emptyFileDestinationPath ); QVERIFY( !checksum.isEmpty() ); //upload @@ -578,7 +579,7 @@ void TestMerginApi::testEmptyFileUploadDownload() downloadRemoteProject( mApi, mUsername, projectName ); // verify it's there and with correct content - QByteArray checksum2 = MerginApi::getChecksum( emptyFileDestinationPath ); + QByteArray checksum2 = CoreUtils::calculateChecksum( emptyFileDestinationPath ); QVERIFY( QFileInfo::exists( emptyFileDestinationPath ) ); QCOMPARE( checksum, checksum2 ); } @@ -612,7 +613,7 @@ void TestMerginApi::testPushAddedFile() QVERIFY( project1.isLocal() && project1.isMergin() ); QCOMPARE( project1.local.localVersion, 1 ); QCOMPARE( project1.mergin.serverVersion, 1 ); - QCOMPARE( project1.mergin.status, ProjectStatus::Modified ); + QCOMPARE( project1.mergin.status, ProjectStatus::NeedsSync ); // upload uploadRemoteProject( mApi, mUsername, projectName ); @@ -670,7 +671,7 @@ void TestMerginApi::testPushRemovedFile() QVERIFY( project1.isLocal() && project1.isMergin() ); QCOMPARE( project1.local.localVersion, 1 ); QCOMPARE( project1.mergin.serverVersion, 1 ); - QCOMPARE( project1.mergin.status, ProjectStatus::Modified ); + QCOMPARE( project1.mergin.status, ProjectStatus::NeedsSync ); // upload changes @@ -726,7 +727,7 @@ void TestMerginApi::testPushModifiedFile() QVERIFY( project1.isLocal() && project1.isMergin() ); QCOMPARE( project1.local.localVersion, 1 ); QCOMPARE( project1.mergin.serverVersion, 1 ); - QCOMPARE( project1.mergin.status, ProjectStatus::Modified ); + QCOMPARE( project1.mergin.status, ProjectStatus::NeedsSync ); // upload uploadRemoteProject( mApi, mUsername, projectName ); @@ -784,6 +785,7 @@ void TestMerginApi::testPushNoChanges() QCOMPARE( project2.mergin.status, ProjectStatus::UpToDate ); QCOMPARE( MerginApi::localProjectChanges( projectDir ), ProjectDiff() ); + QVERIFY( !MerginApi::hasLocalProjectChanges( projectDir ) ); } void TestMerginApi::testUpdateAddedFile() @@ -821,7 +823,7 @@ void TestMerginApi::testUpdateAddedFile() QVERIFY( project1.isLocal() && project1.isMergin() ); QCOMPARE( project1.local.localVersion, 1 ); QCOMPARE( project1.mergin.serverVersion, 2 ); - QCOMPARE( project1.mergin.status, ProjectStatus::OutOfDate ); + QCOMPARE( project1.mergin.status, ProjectStatus::NeedsSync ); // now try to update downloadRemoteProject( mApi, mUsername, projectName ); @@ -1122,6 +1124,7 @@ void TestMerginApi::testDiffUpload() QVERIFY( QFileInfo::exists( projectDir + "/.mergin/base.gpkg" ) ); QCOMPARE( MerginApi::localProjectChanges( projectDir ), ProjectDiff() ); // no local changes expected + QVERIFY( !MerginApi::hasLocalProjectChanges( projectDir ) ); // replace gpkg with a new version with a modified geometry QFile::remove( projectDir + "/base.gpkg" ); @@ -1131,6 +1134,7 @@ void TestMerginApi::testDiffUpload() ProjectDiff expectedDiff; expectedDiff.localUpdated = QSet() << "base.gpkg"; QCOMPARE( diff, expectedDiff ); + QVERIFY( MerginApi::hasLocalProjectChanges( projectDir ) ); GeodiffUtils::ChangesetSummary expectedSummary; expectedSummary["simple"] = GeodiffUtils::TableSummary( 0, 1, 0 ); @@ -1141,6 +1145,7 @@ void TestMerginApi::testDiffUpload() uploadRemoteProject( mApi, mUsername, projectName ); QCOMPARE( MerginApi::localProjectChanges( projectDir ), ProjectDiff() ); // no local changes expected + QVERIFY( !MerginApi::hasLocalProjectChanges( projectDir ) ); } void TestMerginApi::testDiffSubdirsUpload() @@ -1156,7 +1161,7 @@ void TestMerginApi::testDiffSubdirsUpload() QVERIFY( QFileInfo::exists( projectDir + "/.mergin/" + base ) ); QCOMPARE( MerginApi::localProjectChanges( projectDir ), ProjectDiff() ); // no local changes expected - + QVERIFY( !MerginApi::hasLocalProjectChanges( projectDir ) ); // replace gpkg with a new version with a modified geometry QFile::remove( projectDir + "/" + base ); @@ -1166,6 +1171,7 @@ void TestMerginApi::testDiffSubdirsUpload() ProjectDiff expectedDiff; expectedDiff.localUpdated = QSet() << base ; QCOMPARE( diff, expectedDiff ); + QVERIFY( MerginApi::hasLocalProjectChanges( projectDir ) ); GeodiffUtils::ChangesetSummary expectedSummary; expectedSummary["simple"] = GeodiffUtils::TableSummary( 0, 1, 0 ); @@ -1176,6 +1182,7 @@ void TestMerginApi::testDiffSubdirsUpload() uploadRemoteProject( mApi, mUsername, projectName ); QCOMPARE( MerginApi::localProjectChanges( projectDir ), ProjectDiff() ); // no local changes expected + QVERIFY( !MerginApi::hasLocalProjectChanges( projectDir ) ); } void TestMerginApi::testDiffUpdateBasic() @@ -1193,6 +1200,7 @@ void TestMerginApi::testDiffUpdateBasic() QVERIFY( QFileInfo::exists( projectDir + "/.mergin/base.gpkg" ) ); QCOMPARE( MerginApi::localProjectChanges( projectDir ), ProjectDiff() ); // no local changes expected + QVERIFY( !MerginApi::hasLocalProjectChanges( projectDir ) ); QgsVectorLayer *vl0 = new QgsVectorLayer( projectDir + "/base.gpkg|layername=simple", "base", "ogr" ); QVERIFY( vl0->isValid() ); @@ -1222,6 +1230,7 @@ void TestMerginApi::testDiffUpdateBasic() delete vl; QCOMPARE( MerginApi::localProjectChanges( projectDir ), ProjectDiff() ); // no local changes expected + QVERIFY( !MerginApi::hasLocalProjectChanges( projectDir ) ); QVERIFY( !GeodiffUtils::hasPendingChanges( projectDir, "base.gpkg" ) ); } @@ -1241,6 +1250,7 @@ void TestMerginApi::testDiffUpdateWithRebase() QVERIFY( QFileInfo::exists( projectDir + "/.mergin/base.gpkg" ) ); QCOMPARE( MerginApi::localProjectChanges( projectDir ), ProjectDiff() ); // no local changes expected + QVERIFY( !MerginApi::hasLocalProjectChanges( projectDir ) ); // // download with mApiExtra + modify + upload @@ -1274,6 +1284,7 @@ void TestMerginApi::testDiffUpdateWithRebase() ProjectDiff expectedDiff; expectedDiff.localUpdated = QSet() << "base.gpkg"; QCOMPARE( diff, expectedDiff ); + QVERIFY( MerginApi::hasLocalProjectChanges( projectDir ) ); // check that geodiff knows there was one added feature GeodiffUtils::ChangesetSummary expectedSummary; @@ -1297,6 +1308,7 @@ void TestMerginApi::testDiffUpdateWithRebase() // like before the update - there should be locally modified base.gpkg with the changes we did QCOMPARE( MerginApi::localProjectChanges( projectDir ), expectedDiff ); QCOMPARE( GeodiffUtils::parseChangesetSummary( changes ), expectedSummary ); + QVERIFY( MerginApi::hasLocalProjectChanges( projectDir ) ); } void TestMerginApi::testDiffUpdateWithRebaseFailed() @@ -1317,6 +1329,7 @@ void TestMerginApi::testDiffUpdateWithRebaseFailed() QVERIFY( QFileInfo::exists( projectDir + "/.mergin/base.gpkg" ) ); QCOMPARE( MerginApi::localProjectChanges( projectDir ), ProjectDiff() ); // no local changes expected + QVERIFY( !MerginApi::hasLocalProjectChanges( projectDir ) ); // // download with mApiExtra + modify + upload @@ -1342,6 +1355,7 @@ void TestMerginApi::testDiffUpdateWithRebaseFailed() expectedDiff.localUpdated = QSet() << "base.gpkg"; qDebug() << diff.dump(); QCOMPARE( diff, expectedDiff ); + QVERIFY( MerginApi::hasLocalProjectChanges( projectDir ) ); // check that geodiff knows there was one added feature QString changes = GeodiffUtils::diffableFilePendingChanges( projectDir, "base.gpkg", true ); @@ -1367,6 +1381,7 @@ void TestMerginApi::testDiffUpdateWithRebaseFailed() ProjectDiff expectedDiffFinal; expectedDiffFinal.localAdded = QSet() << conflictFilename; QCOMPARE( MerginApi::localProjectChanges( projectDir ), expectedDiffFinal ); + QVERIFY( MerginApi::hasLocalProjectChanges( projectDir ) ); } void TestMerginApi::testUpdateWithDiffs() @@ -1384,6 +1399,7 @@ void TestMerginApi::testUpdateWithDiffs() QVERIFY( QFileInfo::exists( projectDir + "/.mergin/base.gpkg" ) ); QCOMPARE( MerginApi::localProjectChanges( projectDir ), ProjectDiff() ); // no local changes expected + QVERIFY( !MerginApi::hasLocalProjectChanges( projectDir ) ); // // download with mApiExtra + modify + upload @@ -1414,6 +1430,7 @@ void TestMerginApi::testUpdateWithDiffs() delete vl; QCOMPARE( MerginApi::localProjectChanges( projectDir ), ProjectDiff() ); + QVERIFY( !MerginApi::hasLocalProjectChanges( projectDir ) ); QVERIFY( !GeodiffUtils::hasPendingChanges( projectDir, "base.gpkg" ) ); } diff --git a/app/test/testprojectchecksumcache.cpp b/app/test/testprojectchecksumcache.cpp new file mode 100644 index 000000000..b7e3ca5f9 --- /dev/null +++ b/app/test/testprojectchecksumcache.cpp @@ -0,0 +1,138 @@ +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#include "testprojectchecksumcache.h" +#include "projectchecksumcache.h" +#include "coreutils.h" +#include "testutils.h" +#include "inpututils.h" + +#include +#include + +TestProjectChecksumCache::TestProjectChecksumCache() = default; + +TestProjectChecksumCache::~TestProjectChecksumCache() = default; + +void TestProjectChecksumCache::init() +{ +} + +void TestProjectChecksumCache::cleanup() +{ +} + +void TestProjectChecksumCache::testFilesCheckum() +{ + QString projectName = QStringLiteral( "testFilesCheckum" ); + QString projectDir = QDir::tempPath() + "/" + projectName; + + InputUtils::cpDir( TestUtils::testDataDir() + "/planes", projectDir ); + InputUtils::copyFile( TestUtils::testDataDir() + "/photo.jpg", projectDir + "/bigfile.jpg" ); + + QString cacheFilePath = projectDir + "/.mergin/checksum.cache" ; + QString checksumDirectTxt1 = CoreUtils::calculateChecksum( projectDir + "/lines.qml" ); + QVERIFY( !checksumDirectTxt1.isEmpty() ); + + QElapsedTimer timer; + timer.start(); + QString checksumDirectBigFile = CoreUtils::calculateChecksum( projectDir + "/bigfile.jpg" ); + qint64 elapsedForChecksumDirectBigFile = timer.elapsed(); + QVERIFY( !checksumDirectBigFile.isEmpty() ); + + { + // Cold start - delete cache file + InputUtils::removeFile( cacheFilePath ); + ProjectChecksumCache cache( projectDir ); + QCOMPARE( cache.cacheFilePath(), cacheFilePath ); + + // Test gpkg + QString checksumDirectGpkg = CoreUtils::calculateChecksum( projectDir + "/constraint-layers.gpkg" ); + QCOMPARE( checksumDirectGpkg, "c81be103072ecea025ff92a813916db8e42b7bbb" ); + QString checksumFromCacheGpkg = cache.get( "constraint-layers.gpkg" ); + QCOMPARE( checksumDirectGpkg, checksumFromCacheGpkg ); + + // Test non-existent file + InputUtils::removeFile( projectDir + "/photo.jpg" ); + QVERIFY( cache.get( "photo.jpg" ).isEmpty() ); + + // Test text file + QString checksumFromCacheTxt = cache.get( "lines.qml" ); + QCOMPARE( checksumDirectTxt1, checksumFromCacheTxt ); + + // Test photo - big file + QString checksumFromCacheBigFile = cache.get( "bigfile.jpg" ); + QCOMPARE( checksumDirectBigFile, checksumFromCacheBigFile ); + } + + // Test that cache is saved + QVERIFY( QFileInfo( cacheFilePath ).exists() ); + QDateTime cacheModifiedTime = QFileInfo( cacheFilePath ).lastModified(); + + // Modify txt file, remove constraint-layers.gpkg file and add photo file, do not touch bigfile.jpg + InputUtils::removeFile( projectDir + "/constraint-layers.gpkg" ); + InputUtils::copyFile( TestUtils::testDataDir() + "/photo.jpg", projectDir + "/photo.jpg" ); + QString checksumDirectPhoto = CoreUtils::calculateChecksum( projectDir + "/photo.jpg" ); + QVERIFY( !checksumDirectPhoto.isEmpty() ); + + QFile f( projectDir + "/lines.qml" ); + if ( f.open( QIODevice::WriteOnly ) ) + { + QTextStream stream( &f ); + stream << "something really really cool"; + f.close(); + } + QString checksumDirectTxt2 = CoreUtils::calculateChecksum( projectDir + "/lines.qml" ); + QVERIFY( !checksumDirectTxt2.isEmpty() ); + QVERIFY( checksumDirectTxt1 != checksumDirectTxt2 ); + + { + // Start with existent cache file + ProjectChecksumCache cache( projectDir ); + + // Test non-existent gpkg - NOT taken from previous cache + QVERIFY( cache.get( "constraint-layers.gpkg" ).isEmpty() ); + + // Test new file - NOT taken from previous cache + QString checksumDirectPhoto = CoreUtils::calculateChecksum( projectDir + "/photo.jpg" ); + QString checksumFromCachePhoto = cache.get( "photo.jpg" ); + QCOMPARE( checksumDirectPhoto, checksumFromCachePhoto ); + + // Test modified file - NOT taken from previous cache + QString checksumFromCacheTxt = cache.get( "lines.qml" ); + QCOMPARE( checksumFromCacheTxt, checksumDirectTxt2 ); + + // Test bigfile - checksum taken from previous cache! + // time should be faster than when calculated directly (let say at least 2 times) + QElapsedTimer timer2; + timer2.start(); + QString checksumFromCacheBigFile = cache.get( "bigfile.jpg" ); + qint64 elapsedTimeFromCache = timer2.elapsed(); + QVERIFY( elapsedTimeFromCache * 2 < elapsedForChecksumDirectBigFile ); + QCOMPARE( checksumDirectBigFile, checksumFromCacheBigFile ); + } + + // Test that cache is re-saved + QVERIFY( QFileInfo( cacheFilePath ).exists() ); + QDateTime cacheModifiedTime2 = QFileInfo( cacheFilePath ).lastModified(); + QVERIFY( cacheModifiedTime != cacheModifiedTime2 ); + + { + // Start with existent cache file + ProjectChecksumCache cache( projectDir ); + + // Test geo gpkg + QString checksumFromCacheGeoGpkg = cache.get( "bigfile.jpg" ); + QCOMPARE( checksumDirectBigFile, checksumFromCacheGeoGpkg ); + } + // Test that cache is NOT re-saved + QVERIFY( QFileInfo( cacheFilePath ).exists() ); + QDateTime cacheModifiedTime3 = QFileInfo( cacheFilePath ).lastModified(); + QCOMPARE( cacheModifiedTime2, cacheModifiedTime3 ); +} diff --git a/app/test/testprojectchecksumcache.h b/app/test/testprojectchecksumcache.h new file mode 100644 index 000000000..86a8be929 --- /dev/null +++ b/app/test/testprojectchecksumcache.h @@ -0,0 +1,29 @@ +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#ifndef TESTPROJECTCHECKSUMCACHE_H +#define TESTPROJECTCHECKSUMCACHE_H + +#include + +class TestProjectChecksumCache : public QObject +{ + Q_OBJECT + public: + explicit TestProjectChecksumCache( ); + ~TestProjectChecksumCache(); + + private slots: + void init(); + void cleanup(); + + void testFilesCheckum(); +}; + +#endif // TESTPROJECTCHECKSUMCACHE_H diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index b0fe5733a..b9a72fe0c 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -14,6 +14,7 @@ set(MM_CORE_SRCS merginprojectmetadata.cpp project.cpp geodiffutils.cpp + projectchecksumcache.cpp ) set(MM_CORE_HDRS @@ -32,6 +33,7 @@ set(MM_CORE_HDRS merginprojectmetadata.h project.h geodiffutils.h + projectchecksumcache.h ) if (USE_MM_SERVER_API_KEY) diff --git a/core/coreutils.cpp b/core/coreutils.cpp index b5d6a0ad2..4a0cd72b8 100644 --- a/core/coreutils.cpp +++ b/core/coreutils.cpp @@ -16,13 +16,15 @@ #include #include #include +#include +#include #include "qcoreapplication.h" -#include "merginapi.h" const QString CoreUtils::LOG_TO_DEVNULL = QStringLiteral(); const QString CoreUtils::LOG_TO_STDOUT = QStringLiteral( "TO_STDOUT" ); QString CoreUtils::sLogFile = CoreUtils::LOG_TO_DEVNULL; +int CoreUtils::CHECKSUM_CHUNK_SIZE = 65536; QString CoreUtils::appInfo() { @@ -133,39 +135,6 @@ void CoreUtils::appendLog( const QByteArray &data, const QString &path ) } } -QDateTime CoreUtils::getLastModifiedFileDateTime( const QString &path ) -{ - QDateTime lastModified; - QDirIterator it( path, QStringList() << QStringLiteral( "*" ), QDir::Files, QDirIterator::Subdirectories ); - while ( it.hasNext() ) - { - it.next(); - if ( !MerginApi::isInIgnore( it.fileInfo() ) ) - { - if ( it.fileInfo().lastModified() > lastModified ) - { - lastModified = it.fileInfo().lastModified(); - } - } - } - return lastModified.toUTC(); -} - -int CoreUtils::getProjectFilesCount( const QString &path ) -{ - int count = 0; - QDirIterator it( path, QStringList() << QStringLiteral( "*" ), QDir::Files, QDirIterator::Subdirectories ); - while ( it.hasNext() ) - { - it.next(); - if ( !MerginApi::isInIgnore( it.fileInfo() ) ) - { - count++; - } - } - return count; -} - QString CoreUtils::findUniquePath( const QString &path ) { QFileInfo originalPath( path ); @@ -195,6 +164,25 @@ QString CoreUtils::findUniquePath( const QString &path ) return uniquePath; } +QByteArray CoreUtils::calculateChecksum( const QString &filePath ) +{ + QFile f( filePath ); + if ( f.open( QFile::ReadOnly ) ) + { + QCryptographicHash hash( QCryptographicHash::Sha1 ); + QByteArray chunk = f.read( CHECKSUM_CHUNK_SIZE ); + while ( !chunk.isEmpty() ) + { + hash.addData( chunk ); + chunk = f.read( CHECKSUM_CHUNK_SIZE ); + } + f.close(); + return hash.result().toHex(); + } + + return QByteArray(); +} + QString CoreUtils::createUniqueProjectDirectory( const QString &baseDataDir, const QString &projectName ) { QString projectDirPath = findUniquePath( baseDataDir + "/" + projectName ); diff --git a/core/coreutils.h b/core/coreutils.h index 9bb10513e..04db583ec 100644 --- a/core/coreutils.h +++ b/core/coreutils.h @@ -38,8 +38,13 @@ class CoreUtils static QString downloadInProgressFilePath( const QString &projectDir ); static QString uuidWithoutBraces( const QUuid &uuid ); - static QDateTime getLastModifiedFileDateTime( const QString &path ); - static int getProjectFilesCount( const QString &path ); + + /** + * Returns Sha1 checksum of file (no-caching) + * This is potentially resourcing-costly operation + * \param filePath full path to the file on disk + */ + static QByteArray calculateChecksum( const QString &filePath ); /** * Returns given path if it does not exist yet, otherwise adds a number to the path in format: @@ -90,6 +95,8 @@ class CoreUtils private: static QString sLogFile; + static int CHECKSUM_CHUNK_SIZE; + static void appendLog( const QByteArray &data, const QString &path ); }; diff --git a/core/merginapi.cpp b/core/merginapi.cpp index 58b9ac93d..799aa1fc9 100644 --- a/core/merginapi.cpp +++ b/core/merginapi.cpp @@ -18,7 +18,9 @@ #include #include #include +#include +#include "projectchecksumcache.h" #include "coreutils.h" #include "geodiffutils.h" #include "localprojectsmanager.h" @@ -210,6 +212,19 @@ QString MerginApi::listProjectsByName( const QStringList &projectNames ) return QLatin1String(); } + const int listProjectsByNameApiLimit = 50; + QStringList projectNamesToRequest( projectNames ); + + if ( projectNamesToRequest.count() > listProjectsByNameApiLimit ) + { + CoreUtils::log( "list projects by name", QStringLiteral( "Too many local projects: " ) + QString::number( projectNames.count(), 'f', 0 ) ); + const int projectsToRemoveCount = projectNames.count() - listProjectsByNameApiLimit; + QString msg = tr( "Please remove some projects as the app currently\nonly allows up to %1 downloaded projects." ).arg( listProjectsByNameApiLimit ); + notify( msg ); + projectNamesToRequest.erase( projectNamesToRequest.begin() + listProjectsByNameApiLimit, projectNamesToRequest.end() ); + Q_ASSERT( projectNamesToRequest.count() == listProjectsByNameApiLimit ); + } + // Authentification is optional in this case, as there might be public projects without the need to be logged in. // We only want to include auth token when user is logged in. // User's token, however, might have already expired, so let's just refresh it. @@ -218,7 +233,7 @@ QString MerginApi::listProjectsByName( const QStringList &projectNames ) // construct JSON body QJsonDocument body; QJsonObject projects; - QJsonArray projectsArr = QJsonArray::fromStringList( projectNames ); + QJsonArray projectsArr = QJsonArray::fromStringList( projectNamesToRequest ); projects.insert( "projects", projectsArr ); body.setObject( projects ); @@ -1474,6 +1489,16 @@ ProjectDiff MerginApi::localProjectChanges( const QString &projectDir ) return compareProjectFiles( projectMetadata.files, projectMetadata.files, localFiles, projectDir, config.isValid, config ); } +bool MerginApi::hasLocalProjectChanges( const QString &projectDir ) +{ + MerginProjectMetadata projectMetadata = MerginProjectMetadata::fromCachedJson( projectDir + "/" + sMetadataFile ); + QList localFiles = getLocalProjectFiles( projectDir + "/" ); + + MerginConfig config = MerginConfig::fromFile( projectDir + "/" + sMerginConfigFile ); + + return hasLocalChanges( projectMetadata.files, localFiles, projectDir ); +} + QString MerginApi::getTempProjectDir( const QString &projectFullName ) { return mDataDir + "/" + TEMP_FOLDER + projectFullName; @@ -1582,21 +1607,29 @@ QString MerginApi::merginUserName() const QList MerginApi::getLocalProjectFiles( const QString &projectPath ) { + QElapsedTimer timer; + timer.start(); + QList merginFiles; + ProjectChecksumCache checksumCache( projectPath ); + QSet localFiles = listFiles( projectPath ); for ( QString p : localFiles ) { - MerginFile file; - QByteArray localChecksumBytes = getChecksum( projectPath + p ); - QString localChecksum = QString::fromLatin1( localChecksumBytes.data(), localChecksumBytes.size() ); - file.checksum = localChecksum; + file.checksum = checksumCache.get( p ); file.path = p; QFileInfo info( projectPath + p ); file.size = info.size(); file.mtime = info.lastModified(); merginFiles.append( file ); } + + qint64 elapsed = timer.elapsed(); + if ( elapsed > 100 ) + { + CoreUtils::log( "Local File", QStringLiteral( "It took %1 ms to create MerginFiles for %2 local files for %3." ).arg( elapsed ).arg( localFiles.count() ).arg( projectPath ) ); + } return merginFiles; } @@ -2507,7 +2540,7 @@ void MerginApi::pushInfoReplyFinished() if ( geodiffRes == GEODIFF_SUCCESS ) { - QByteArray checksumDiff = getChecksum( diffPath ); + QByteArray checksumDiff = CoreUtils::calculateChecksum( diffPath ); // TODO: this is ugly. our basefile may not need to have the same checksum as the server's // basefile (because each of them have applied the diff independently) so we have to fake it @@ -2778,6 +2811,67 @@ void MerginApi::getWorkspaceInfoReplyFinished() r->deleteLater(); } +bool MerginApi::hasLocalChanges( + const QList &oldServerFiles, + const QList &localFiles, + const QString &projectDir +) +{ + if ( localFiles.count() != oldServerFiles.count() ) + { + return true; + } + + QHash oldServerFilesMap; + + for ( const MerginFile &file : oldServerFiles ) + { + oldServerFilesMap.insert( file.path, file ); + } + + for ( const MerginFile &localFile : localFiles ) + { + QString filePath = localFile.path; + bool hasOldServer = oldServerFilesMap.contains( localFile.path ); + + if ( !hasOldServer ) + { + // L-A + return true; + } + else + { + const QString chkOld = oldServerFilesMap.value( localFile.path ).checksum; + const QString chkLocal = localFile.checksum; + + if ( chkOld != chkLocal ) + { + if ( isFileDiffable( filePath ) ) + { + // we need to do a diff here to figure out whether the file is actually changed or not + // because the real content may be the same although the checksums do not match + // e.g. when GPKG is opened, its header is updated and therefore lastModified timestamp/checksum is updated as well. + if ( GeodiffUtils::hasPendingChanges( projectDir, filePath ) ) + { + // L-U + return true; + } + } + else + { + // L-U + return true; + } + } + } + } + + // We know that the number of local files and old server is the same + // And also that all local files has old file counterpart + // So it is not possible that there is deleted local file at this point. + return false; +} + ProjectDiff MerginApi::compareProjectFiles( const QList &oldServerFiles, const QList &newServerFiles, @@ -2956,11 +3050,13 @@ ProjectDiff MerginApi::compareProjectFiles( oldServerFilesMap.remove( file.path ); } + /* for ( MerginFile file : oldServerFilesMap ) { // R-D/L-D // TODO: need to do anything? } + */ return diff; } @@ -3229,25 +3325,6 @@ bool MerginApi::excludeFromSync( const QString &filePath, const MerginConfig &co return false; } -QByteArray MerginApi::getChecksum( const QString &filePath ) -{ - QFile f( filePath ); - if ( f.open( QFile::ReadOnly ) ) - { - QCryptographicHash hash( QCryptographicHash::Sha1 ); - QByteArray chunk = f.read( CHUNK_SIZE ); - while ( !chunk.isEmpty() ) - { - hash.addData( chunk ); - chunk = f.read( CHUNK_SIZE ); - } - f.close(); - return hash.result().toHex(); - } - - return QByteArray(); -} - QSet MerginApi::listFiles( const QString &path ) { QSet files; diff --git a/core/merginapi.h b/core/merginapi.h index b1d83d989..eedcbebef 100644 --- a/core/merginapi.h +++ b/core/merginapi.h @@ -386,6 +386,7 @@ class MerginApi: public QObject QStringList projectDiffableFiles( const QString &projectFullName ); static ProjectDiff localProjectChanges( const QString &projectDir ); + static bool hasLocalProjectChanges( const QString &projectDir ); /** * Finds project in merginProjects list according its full name. @@ -441,6 +442,23 @@ class MerginApi: public QObject const MerginConfig &lastSyncConfig = MerginConfig() ); + /** + * Finds if project files from two sources are same + * - "old" server version (what was downloaded from server) - read from the project directory's stored metadata + * - local file version (what is currently in the project directory) - created on the fly from the local directory content + * + * The function returns true if: + * - there is any local file not present in "old" server version files + * - there is any local file missing in "old" server version files + * - there is different checksum of any non-diffable file (e.g. CSV file) + * - there is different content of any diffable file (e.g. GeoPackage) + */ + static bool hasLocalChanges( + const QList &oldServerFiles, + const QList &localFiles, + const QString &projectDir + ); + static QList getLocalProjectFiles( const QString &projectPath ); QString apiRoot() const; @@ -689,7 +707,6 @@ class MerginApi: public QObject bool writeData( const QByteArray &data, const QString &path ); void createPathIfNotExists( const QString &filePath ); - static QByteArray getChecksum( const QString &filePath ); static QSet listFiles( const QString &projectPath ); bool validateAuth(); @@ -779,7 +796,6 @@ class MerginApi: public QObject bool mApiSupportsSubscriptions = false; bool mSupportsSelectiveSync = true; - static const int CHUNK_SIZE = 65536; static const int UPLOAD_CHUNK_SIZE; const int PROJECT_PER_PAGE = 50; const QString TEMP_FOLDER = QStringLiteral( ".temp/" ); diff --git a/core/project.cpp b/core/project.cpp index e6edaa2d2..e2efd99fc 100644 --- a/core/project.cpp +++ b/core/project.cpp @@ -44,15 +44,15 @@ ProjectStatus::Status ProjectStatus::projectStatus( const Project &project ) return ProjectStatus::NoVersion; } - if ( ProjectStatus::hasLocalChanges( project.local ) ) + // Version is lower than latest one, last sync also before updated + if ( project.local.localVersion < project.mergin.serverVersion ) { - return ProjectStatus::Modified; + return ProjectStatus::NeedsSync; } - // Version is lower than latest one, last sync also before updated - if ( project.local.localVersion < project.mergin.serverVersion ) + if ( ProjectStatus::hasLocalChanges( project.local ) ) { - return ProjectStatus::OutOfDate; + return ProjectStatus::NeedsSync; } return ProjectStatus::UpToDate; @@ -62,29 +62,11 @@ bool ProjectStatus::hasLocalChanges( const LocalProject &project ) { QString metadataFilePath = project.projectDir + "/" + MerginApi::sMetadataFile; + // If the project does not have metadata file, there are local changes if ( !QFile::exists( metadataFilePath ) ) { - // If the project does not have metadata file, there are local changes return true; } - // Check if something has locally changed after last sync with server - QDateTime lastModified = CoreUtils::getLastModifiedFileDateTime( project.projectDir ); - - QDateTime lastSync = QFileInfo( metadataFilePath ).lastModified().toUTC(); - MerginProjectMetadata metadata = MerginProjectMetadata::fromCachedJson( metadataFilePath ); - - int filesCount = CoreUtils::getProjectFilesCount( project.projectDir ); - - if ( lastSync < lastModified || metadata.files.count() != filesCount ) - { - // When GPKG is opened, its header is updated and therefore lastModified timestamp is updated as well. - // Double check if something has really changed - ProjectDiff diff = MerginApi::localProjectChanges( project.projectDir ); - - if ( !diff.localAdded.isEmpty() || !diff.localUpdated.isEmpty() || !diff.localDeleted.isEmpty() ) - return true; - } - - return false; + return MerginApi::hasLocalProjectChanges( project.projectDir ); } diff --git a/core/project.h b/core/project.h index 8ac55e088..24830ac83 100644 --- a/core/project.h +++ b/core/project.h @@ -25,9 +25,7 @@ namespace ProjectStatus { NoVersion, //!< the project is not downloaded UpToDate, //!< both server and local copy are in sync with no extra modifications - OutOfDate, //!< server has newer version than what is available locally (but the project is not modified locally) - Modified //!< there are some local modifications in the project that need to be pushed (note: also server may have newer version) - // Maybe orphaned state in future + NeedsSync, //!< server has newer version than what is available locally and/or the project is modified locally }; Q_ENUM_NS( Status ) diff --git a/core/projectchecksumcache.cpp b/core/projectchecksumcache.cpp new file mode 100644 index 000000000..896b9a62a --- /dev/null +++ b/core/projectchecksumcache.cpp @@ -0,0 +1,106 @@ +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#include +#include +#include +#include + +#include "projectchecksumcache.h" +#include "coreutils.h" +#include "merginapi.h" + +const QString ProjectChecksumCache::sCacheFile = QStringLiteral( "checksum.cache" ); + +QString ProjectChecksumCache::cacheFilePath() const +{ + return cacheDirPath() + "/" + sCacheFile; +} + +QString ProjectChecksumCache::cacheDirPath() const +{ + return mProjectDir + "/" + MerginApi::sMetadataFolder; +} + +ProjectChecksumCache::ProjectChecksumCache( const QString &projectDir ) + : mProjectDir( projectDir ) +{ + QFile f( cacheFilePath() ); + + if ( f.open( QIODevice::ReadOnly ) ) + { + QDataStream stream( &f ); + stream.setVersion( QDataStream::Qt_6_5 ); + QString path; + QString checksum; + QDateTime mtime; + CacheValue entry; + + while ( stream.atEnd() == false ) + { + stream >> path >> checksum >> mtime; + entry.checksum = checksum; + entry.mtime = mtime; + mCache.insert( path, entry ); + } + } +} + +ProjectChecksumCache::~ProjectChecksumCache() +{ + if ( !mCacheModified ) + return; + + // Make sure the directory exists + QDir dir; + if ( !dir.exists( cacheDirPath() ) ) + dir.mkpath( cacheDirPath() ); + + QFile f( cacheFilePath() ); + if ( f.open( QIODevice::WriteOnly ) ) // implies Truncate + { + QDataStream stream( &f ); + stream.setVersion( QDataStream::Qt_6_5 ); + + for ( auto it = mCache.constBegin(); it != mCache.constEnd(); ++it ) + { + stream << it.key() << it.value().checksum << it.value().mtime; + } + } + else + { + CoreUtils::log( "projectchecksumcache", QStringLiteral( "Unable to save cache %1" ).arg( cacheFilePath() ) ); + } +} + +QString ProjectChecksumCache::get( const QString &path ) +{ + QDateTime localLastModified = QFileInfo( mProjectDir + "/" + path ).lastModified(); + + auto match = mCache.find( path ); + + if ( match != mCache.end() ) + { + if ( match.value().mtime == localLastModified ) + { + return match.value().checksum; + } + } + + QByteArray localChecksumBytes = CoreUtils::calculateChecksum( mProjectDir + "/" + path ); + QString localChecksum = QString::fromLatin1( localChecksumBytes.data(), localChecksumBytes.size() ); + + CacheValue entry; + entry.checksum = localChecksum; + entry.mtime = localLastModified; + mCache.insert( path, entry ); + mCacheModified = true; + + return localChecksum; +} diff --git a/core/projectchecksumcache.h b/core/projectchecksumcache.h new file mode 100644 index 000000000..ba69b88d0 --- /dev/null +++ b/core/projectchecksumcache.h @@ -0,0 +1,62 @@ +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ + +#ifndef PROJECTCHECKSUMCACHE_H +#define PROJECTCHECKSUMCACHE_H + +#include +#include +#include +#include + +#include "inputconfig.h" + +#if defined(INPUT_TEST) +class TestProjectChecksumCache; +#endif + +/** + * Calculates the checksums of local files and store the results in the local binary file + */ +class ProjectChecksumCache +{ + public: + ProjectChecksumCache( const QString &projectDir ); + ~ProjectChecksumCache(); + + /** + * Returns Sha1 checksum of file (with-caching) + * Recalculates checksum for an entry not in cache + * \param path relative path of the file to mProjectDir + */ + QString get( const QString &path ); + + //! Name of the file in which the cache for the project is stored + static const QString sCacheFile; + +#if defined(INPUT_TEST) + friend class TestProjectChecksumCache; +#endif + + private: + QString cacheFilePath() const; + QString cacheDirPath() const; + + struct CacheValue + { + QDateTime mtime; //!< associated file modification date when checksum was calculated + QString checksum; //!< calculated checksum + }; + + QString mProjectDir; + QHash mCache; //!< key -> file relative path to mProjectDir + bool mCacheModified = false; +}; + +#endif // PROJECTCHECKSUMCACHE_H diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6572558b0..4e1c7cf5d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -24,6 +24,7 @@ set(MM_TESTS testMapTools testLayerTree testActiveProject + testProjectChecksumCache ) foreach (test ${MM_TESTS})