[LEIP-272] Upload attachments unwrapped, uploader 1.5.0

Task/leip 272 implement specification
cyface-de · Oct 23, 2024 · f3809a5 · f3809a5
2 parents 38a4610 + 68d494d
commit f3809a5
Show file tree

Hide file tree

Showing 8 changed files with 40 additions and 30 deletions.
diff --git a/build.gradle b/build.gradle
@@ -53,7 +53,7 @@ ext {
     // When `android-app` is checkout out, `rootProject.uploaderVersion` in the submodules will point
     // to `android-app/build.gradle` instead of `submodule/build.gradle`.
     cyfaceSerializationVersion = "3.4.2"
-    cyfaceUploaderVersion = "1.4.1"
+    cyfaceUploaderVersion = "1.5.0"
 
     // Android SDK versions
     minSdkVersion = 21 // device support

diff --git a/persistence/src/main/kotlin/de/cyface/persistence/serialization/AttachmentSerializer.kt b/persistence/src/main/kotlin/de/cyface/persistence/serialization/AttachmentSerializer.kt
@@ -28,8 +28,6 @@ import java.nio.file.Files
  * Serializes a [Attachment] in the [MeasurementSerializer.TRANSFER_FILE_FORMAT_VERSION].
  *
  * @author Armin Schnabel
- * @version 1.0.0
- * @since 7.10.0
  */
 class AttachmentSerializer {
     /**

diff --git a/persistence/src/main/kotlin/de/cyface/persistence/serialization/MeasurementSerializer.kt b/persistence/src/main/kotlin/de/cyface/persistence/serialization/MeasurementSerializer.kt
@@ -185,6 +185,11 @@ class MeasurementSerializer {
      * layer serialized in the [MeasurementSerializer.TRANSFER_FILE_FORMAT_VERSION] format, ready to be
      * transferred.
      *
+     * Attention:
+     * We don't wrap the attachments in the `cyf` wrapper, as:
+     * - Most our project currently prefer the plain JPG, CSV, ZIP, etc. formats
+     * - We have a version in meta data, and currently have version 1 for attachment files format.
+     *
      * No compression is used as we're mostly transferring JPG files right now which are pre-compressed.
      *
      * @param fileOutputStream the `FileInputStream` to write the compressed data to

diff --git a/persistence/src/main/kotlin/de/cyface/persistence/serialization/TransferFileSerializer.kt b/persistence/src/main/kotlin/de/cyface/persistence/serialization/TransferFileSerializer.kt
@@ -29,7 +29,6 @@ import de.cyface.persistence.content.AbstractCyfaceTable.Companion.DATABASE_QUER
 import de.cyface.persistence.model.Attachment
 import de.cyface.persistence.model.Measurement
 import de.cyface.protos.model.Event
-import de.cyface.protos.model.File.FileType
 import de.cyface.protos.model.LocationRecords
 import de.cyface.protos.model.MeasurementBytes
 import de.cyface.serializer.DataSerializable
@@ -40,13 +39,12 @@ import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 import java.io.BufferedOutputStream
 import java.io.IOException
+import java.util.Locale
 
 /**
  * Serializes [MeasurementSerializer.TRANSFER_FILE_FORMAT_VERSION] files.
  *
  * @author Armin Schnabel
- * @version 3.1.0
- * @since 5.0.0
  */
 object TransferFileSerializer {
     /**
@@ -253,13 +251,16 @@ object TransferFileSerializer {
      * Implements the core algorithm of loading data of a [Attachment] from the [PersistenceLayer]
      * and serializing it into an array of bytes, ready to be transferred.
      *
-     * We use the {@param loader} to access the measurement data. FIXME?
-     *
      * We assemble the data using a buffer to avoid OOM exceptions.
      *
      * **ATTENTION:** The caller must make sure the {@param bufferedOutputStream} is closed when no longer needed
      * or the app crashes.
      *
+     * Attention:
+     * We don't wrap the attachments in the `cyf` wrapper, as:
+     * - Most our project currently prefer the plain JPG, CSV, ZIP, etc. formats
+     * - We have a version in meta data, and currently have version 1 for attachment files format.
+     *
      * @param bufferedOutputStream The `OutputStream` to which the serialized data should be written. Injecting
      * this allows us to compress the serialized data without the need to write it into a temporary file.
      * We require a [BufferedOutputStream] for performance reasons.
@@ -274,10 +275,17 @@ object TransferFileSerializer {
     ) {
         val attachment = loadAttachment(reference)
 
-        val builder = de.cyface.protos.model.Measurement.newBuilder()
+        // In case we switch back to the cyf wrapper for attachments, we need to adjust the code:
+        // Out Protobuf format only supports one `capturing_log` file, but we collect multiple
+        // files which do not fit the "images" or "video" categories (i.e. CSV, JSON). If we would
+        // upload all attachments in one request, we would need to compress them into one ZIP files.
+        // That is why we added the file format "ZIP" to the Protobuf message definition.
+        // But as we upload each attachment separately, even with the cyf wrapper we should be fine
+        // with one `capturing_log` support, as we can just add this one file as such.
+        // So if you enable the cyf wrapping code below again, make sure all log files are uploaded.
+        /*val builder = de.cyface.protos.model.Measurement.newBuilder()
             .setFormatVersion(MeasurementSerializer.TRANSFER_FILE_FORMAT_VERSION.toInt())
         when (reference.type) {
-            // TODO: zip all attachments
             FileType.JSON, FileType.CSV -> {
                 builder.capturingLog = attachment
             }
@@ -289,28 +297,30 @@ object TransferFileSerializer {
             else -> {
                 throw IllegalArgumentException("Unsupported type: ${reference.type}")
             }
-        }
+        }*/
 
         // Currently loading one image per transfer file into memory (~ 2-5 MB / image).
         // - To load add all high-res image data or video data in the future we cannot use the pre-compiled
         // builder but have to stream the data without loading it into memory to avoid an OOM exception.
         val transferFileHeader = DataSerializable.transferFileHeader()
-        val measurementBytes = builder.build().toByteArray()
+        //val uploadBytes = builder.build().toByteArray()
+        val uploadBytes = attachment.bytes.toByteArray()
         try {
             // The stream must be closed by the caller in a finally catch
             withContext(Dispatchers.IO) {
                 bufferedOutputStream.write(transferFileHeader)
-                bufferedOutputStream.write(measurementBytes)
+                bufferedOutputStream.write(uploadBytes)
                 bufferedOutputStream.flush()
             }
         } catch (e: IOException) {
             throw IllegalStateException(e)
         }
         Log.d(
             TAG, String.format(
+                Locale.getDefault(),
                 "Serialized attachment: %s",
                 DataSerializable.humanReadableSize(
-                    (transferFileHeader.size + measurementBytes.size).toLong(),
+                    (transferFileHeader.size + uploadBytes.size).toLong(),
                     true
                 )
             )

diff --git a/synchronization/src/androidTest/kotlin/de/cyface/synchronization/MockedUploader.kt b/synchronization/src/androidTest/kotlin/de/cyface/synchronization/MockedUploader.kt
@@ -23,6 +23,7 @@ import de.cyface.uploader.UploadProgressListener
 import de.cyface.uploader.Uploader
 import de.cyface.uploader.model.Attachment
 import de.cyface.uploader.model.Measurement
+import de.cyface.uploader.model.Uploadable
 import java.io.File
 import java.net.MalformedURLException
 import java.net.URL
@@ -34,15 +35,15 @@ import java.net.URL
  */
 internal class MockedUploader : Uploader {
 
-    override fun measurementsEndpoint(): URL {
+    override fun measurementsEndpoint(uploadable: Uploadable): URL {
         return try {
             URL("https://mocked.cyface.de/api/v123/measurements")
         } catch (e: MalformedURLException) {
             throw IllegalStateException(e)
         }
     }
 
-    override fun attachmentsEndpoint(deviceId: String, measurementId: Long): URL {
+    override fun attachmentsEndpoint(uploadable: Uploadable): URL {
         return try {
             URL("https://mocked.cyface.de/api/v123/measurements/did/mid/attachments")
         } catch (e: MalformedURLException) {

diff --git a/synchronization/src/androidTest/kotlin/de/cyface/synchronization/SyncPerformerTest.kt b/synchronization/src/androidTest/kotlin/de/cyface/synchronization/SyncPerformerTest.kt
@@ -57,6 +57,7 @@ import de.cyface.uploader.exception.UploadFailed
 import de.cyface.uploader.exception.UploadSessionExpired
 import de.cyface.uploader.model.Attachment
 import de.cyface.uploader.model.MeasurementIdentifier
+import de.cyface.uploader.model.Uploadable
 import de.cyface.uploader.model.metadata.ApplicationMetaData
 import de.cyface.uploader.model.metadata.AttachmentMetaData
 import de.cyface.uploader.model.metadata.DeviceMetaData
@@ -309,12 +310,12 @@ class SyncPerformerTest {
 
         // Mock the actual post request
         val mockedUploader = object : Uploader {
-            override fun measurementsEndpoint(): URL {
+            override fun measurementsEndpoint(uploadable: Uploadable): URL {
                 return URL("https://mocked.cyface.de/api/v123/measurements")
             }
 
-            override fun attachmentsEndpoint(deviceId: String, measurementId: Long): URL {
-                return URL("https://mocked.cyface.de/api/v123/measurements/$deviceId/$measurementId/attachments")
+            override fun attachmentsEndpoint(uploadable: Uploadable): URL {
+                return URL("https://mocked.cyface.de/api/v123/measurements/${uploadable.deviceId()}/${uploadable.measurementId()}/attachments")
             }
 
             override fun uploadMeasurement(

diff --git a/synchronization/src/main/kotlin/de/cyface/synchronization/SyncAdapter.kt b/synchronization/src/main/kotlin/de/cyface/synchronization/SyncAdapter.kt
@@ -37,7 +37,6 @@ import de.cyface.persistence.exception.NoSuchMeasurementException
 import de.cyface.persistence.model.AttachmentStatus
 import de.cyface.persistence.model.Measurement
 import de.cyface.persistence.model.MeasurementStatus
-import de.cyface.persistence.model.ParcelableGeoLocation
 import de.cyface.persistence.serialization.MeasurementSerializer
 import de.cyface.protos.model.File.FileType
 import de.cyface.synchronization.ErrorHandler.ErrorCode
@@ -301,6 +300,7 @@ class SyncAdapter private constructor(
 
                         if (isSyncRequestAborted(account, authority)) return
 
+                        @Suppress("SpellCheckingInspection")
                         val indexWithinMeasurement = 1 + syncedAttachments + attachmentIndex // ccyf is index 0
                         val progressListener = DefaultUploadProgressListener(
                             measurementCount,
@@ -313,6 +313,7 @@ class SyncAdapter private constructor(
                         val attachmentMeta = attachmentMeta(measurementMeta, attachment.id)
                         error = !syncAttachment(
                             attachmentMeta,
+                            attachment.type,
                             syncPerformer,
                             transferTempFile,
                             syncResult,
@@ -433,6 +434,7 @@ class SyncAdapter private constructor(
 
     private suspend fun syncAttachment(
         attachment: Attachment,
+        attachmentType: FileType,
         syncPerformer: SyncPerformer,
         transferFile: File?,
         syncResult: SyncResult,
@@ -456,7 +458,7 @@ class SyncAdapter private constructor(
             } else {
                 val attachmentId = attachment.identifier.attachmentIdentifier
                 val fileName =
-                    "${attachment.identifier.deviceIdentifier}_${attachment.identifier.measurementIdentifier}_$attachmentId.${TRANSFER_FILE_EXTENSION}"
+                    "${attachment.identifier.deviceIdentifier}_${attachment.identifier.measurementIdentifier}_$attachmentId.${attachmentType.name.lowercase()}"
                 val result = syncPerformer.sendData(
                     uploader,
                     syncResult,
@@ -745,11 +747,6 @@ class SyncAdapter private constructor(
          */
         const val MOCK_IS_CONNECTED_TO_RETURN_TRUE = "mocked_periodic_sync_check_false"
 
-        /**
-         * The file extension of the attachment file which is transmitted on synchronization.
-         */
-        private const val TRANSFER_FILE_EXTENSION = "cyf"
-
         /**
          * The file extension of the measurement file which is transmitted on synchronization.
          */

diff --git a/synchronization/src/main/kotlin/de/cyface/synchronization/SyncPerformer.kt b/synchronization/src/main/kotlin/de/cyface/synchronization/SyncPerformer.kt
@@ -101,7 +101,7 @@ internal class SyncPerformer(private val context: Context, private val fromBackg
                 val result = try {
                     when (uploadType) {
                         UploadType.MEASUREMENT ->  {
-                            val endpoint = uploader.measurementsEndpoint()
+                            val endpoint = uploader.measurementsEndpoint(uploadable)
                             Log.i(TAG, "Uploading $fileName to $endpoint.")
                             uploader.uploadMeasurement(
                                 jwtAuthToken,
@@ -113,9 +113,7 @@ internal class SyncPerformer(private val context: Context, private val fromBackg
 
                         UploadType.ATTACHMENT -> {
                             val attachment = uploadable as Attachment
-                            val measurementId = attachment.identifier.measurementIdentifier
-                            val deviceId = uploadable.identifier.deviceIdentifier.toString()
-                            val endpoint = uploader.attachmentsEndpoint(deviceId, measurementId)
+                            val endpoint = uploader.attachmentsEndpoint(uploadable)
                             Log.i(TAG, "Uploading $fileName to $endpoint.")
                             uploader.uploadAttachment(
                                 jwtAuthToken,