Avoid constantly re-downloading same large files from the cloud by fi…

…xing checksum
opengisch · Oct 24, 2024 · 74216ac · 74216ac
1 parent 987c87a
commit 74216ac
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 13 deletions.
diff --git a/src/core/qfieldcloudprojectsmodel.cpp b/src/core/qfieldcloudprojectsmodel.cpp
@@ -1033,24 +1033,24 @@ void QFieldCloudProjectsModel::projectDownload( const QString &projectId )
     const QJsonArray files = payload.value( QStringLiteral( "files" ) ).toArray();
     for ( const QJsonValue &fileValue : files )
     {
-      QJsonObject fileObject = fileValue.toObject();
-      int fileSize = fileObject.value( QStringLiteral( "size" ) ).toInt();
-      QString fileName = fileObject.value( QStringLiteral( "name" ) ).toString();
-      QString projectFileName = QStringLiteral( "%1/%2/%3/%4" ).arg( QFieldCloudUtils::localCloudDirectory(), mUsername, projectId, fileName );
-      QString cloudChecksumMd5 = fileObject.value( QStringLiteral( "md5sum" ) ).toString();
-      QString localChecksumMd5 = FileUtils::fileChecksum( projectFileName, QCryptographicHash::Md5 ).toHex();
+      const QJsonObject fileObject = fileValue.toObject();
+      const int fileSize = fileObject.value( QStringLiteral( "size" ) ).toInt();
+      const QString fileName = fileObject.value( QStringLiteral( "name" ) ).toString();
+      const QString projectFileName = QStringLiteral( "%1/%2/%3/%4" ).arg( QFieldCloudUtils::localCloudDirectory(), mUsername, projectId, fileName );
+      const QString cloudChecksum = fileObject.value( QStringLiteral( "md5sum" ) ).toString();
+      const QString localChecksum = FileUtils::fileEtag( projectFileName );
 
       if (
         !fileObject.value( QStringLiteral( "size" ) ).isDouble()
         || fileName.isEmpty()
-        || cloudChecksumMd5.isEmpty() )
+        || cloudChecksum.isEmpty() )
       {
         QgsLogger::debug( QStringLiteral( "Project %1: package in \"files\" list does not contain the expected fields: size(int), name(string), md5sum(string)" ).arg( projectId ) );
         emit projectDownloadFinished( projectId, tr( "Latest package data structure error." ) );
         return;
       }
 
-      if ( cloudChecksumMd5 == localChecksumMd5 )
+      if ( cloudChecksum == localChecksum )
         continue;
 
       project->downloadFileTransfers.insert( fileName, FileTransfer( fileName, fileSize ) );

diff --git a/src/core/utils/fileutils.cpp b/src/core/utils/fileutils.cpp
@@ -174,6 +174,38 @@ QByteArray FileUtils::fileChecksum( const QString &fileName, const QCryptographi
   return QByteArray();
 }
 
+QString FileUtils::fileEtag( const QString &fileName, int partSize )
+{
+  QFile f( fileName );
+  if ( !f.open( QFile::ReadOnly ) )
+    return QString();
+
+  const qint64 fileSize = f.size();
+  QCryptographicHash hash( QCryptographicHash::Md5 );
+  if ( fileSize <= partSize )
+  {
+    if ( hash.addData( &f ) )
+    {
+      return hash.result().toHex();
+    }
+  }
+  else
+  {
+    QByteArray md5SumsData;
+    qint64 readSize = 0;
+    while ( readSize < fileSize )
+    {
+      hash.addData( f.read( partSize ) );
+      md5SumsData += hash.result();
+      hash.reset();
+      readSize += partSize;
+    }
+    hash.addData( md5SumsData );
+    return QStringLiteral( "%1-%2" ).arg( hash.result().toHex() ).arg( readSize / partSize );
+  }
+  return QString();
+}
+
 void FileUtils::restrictImageSize( const QString &imagePath, int maximumWidthHeight )
 {
   if ( !QFileInfo::exists( imagePath ) )

diff --git a/src/core/utils/fileutils.h b/src/core/utils/fileutils.h
@@ -58,15 +58,23 @@ class QFIELD_CORE_EXPORT FileUtils : public QObject
     Q_INVOKABLE void addImageStamp( const QString &imagePath, const QString &text );
 
     static bool copyRecursively( const QString &sourceFolder, const QString &destFolder, QgsFeedback *feedback = nullptr, bool wipeDestFolder = true );
+
     /**
-     * Creates checksum of a file. Returns null QByteArray if cannot be calculated.
-     *
-     * @param fileName file name to get checksum of
-     * @param hashAlgorithm hash algorithm (md5, sha1, sha256 etc)
-     * @return QByteArray checksum
+     * Returns the checksum of a file. An empty QByteArray will be returned if it cannot be calculated.
+     * \param fileName file name to get checksum of
+     * \param hashAlgorithm hash algorithm (md5, sha1, sha256 etc)
+     * \return QByteArray checksum value
      */
     Q_INVOKABLE static QByteArray fileChecksum( const QString &fileName, const QCryptographicHash::Algorithm hashAlgorithm );
 
+    /**
+     * Returns an S3 ETag of a file. An empty string will be returned if it cannot be calculated.
+     * \param fileName file name to get checksum of
+     * \param partSize maximum size used to divide the file content into parts
+     * \return QString Etag value
+     */
+    Q_INVOKABLE static QString fileEtag( const QString &fileName, int partSize = 8 * 1024 * 1024 );
+
   private:
     static int copyRecursivelyPrepare( const QString &sourceFolder, const QString &destFolder, QList<QPair<QString, QString>> &mapping );
 };