From d391a01ad474221d8eabd21687104742a31d35b3 Mon Sep 17 00:00:00 2001 From: Armin Date: Tue, 8 Oct 2024 09:21:57 +0100 Subject: [PATCH 1/5] Upgrade to uploader alpha --- build.gradle | 2 +- .../main/kotlin/de/cyface/synchronization/SyncPerformer.kt | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index 637a5514..58f6c78b 100644 --- a/build.gradle +++ b/build.gradle @@ -53,7 +53,7 @@ ext { // When `android-app` is checkout out, `rootProject.uploaderVersion` in the submodules will point // to `android-app/build.gradle` instead of `submodule/build.gradle`. cyfaceSerializationVersion = "3.4.2" - cyfaceUploaderVersion = "1.4.1" + cyfaceUploaderVersion = "1.5.0-alpha7" // FIXME // Android SDK versions minSdkVersion = 21 // device support diff --git a/synchronization/src/main/kotlin/de/cyface/synchronization/SyncPerformer.kt b/synchronization/src/main/kotlin/de/cyface/synchronization/SyncPerformer.kt index 6a48d728..d4655cb7 100644 --- a/synchronization/src/main/kotlin/de/cyface/synchronization/SyncPerformer.kt +++ b/synchronization/src/main/kotlin/de/cyface/synchronization/SyncPerformer.kt @@ -101,7 +101,7 @@ internal class SyncPerformer(private val context: Context, private val fromBackg val result = try { when (uploadType) { UploadType.MEASUREMENT -> { - val endpoint = uploader.measurementsEndpoint() + val endpoint = uploader.measurementsEndpoint(uploadable) Log.i(TAG, "Uploading $fileName to $endpoint.") uploader.uploadMeasurement( jwtAuthToken, @@ -113,9 +113,7 @@ internal class SyncPerformer(private val context: Context, private val fromBackg UploadType.ATTACHMENT -> { val attachment = uploadable as Attachment - val measurementId = attachment.identifier.measurementIdentifier - val deviceId = uploadable.identifier.deviceIdentifier.toString() - val endpoint = uploader.attachmentsEndpoint(deviceId, measurementId) + val endpoint = uploader.attachmentsEndpoint(uploadable) Log.i(TAG, "Uploading $fileName to $endpoint.") uploader.uploadAttachment( jwtAuthToken, From c8afb338d903d8d517de543bb6eca9569c4190fa Mon Sep 17 00:00:00 2001 From: Armin Date: Wed, 23 Oct 2024 11:05:52 +0100 Subject: [PATCH 2/5] [LEIP-277] Upload attachments without cyf wrapper --- .../serialization/AttachmentSerializer.kt | 2 -- .../serialization/MeasurementSerializer.kt | 5 +++++ .../serialization/TransferFileSerializer.kt | 21 ++++++++++--------- .../de/cyface/synchronization/SyncAdapter.kt | 11 ++++------ 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/persistence/src/main/kotlin/de/cyface/persistence/serialization/AttachmentSerializer.kt b/persistence/src/main/kotlin/de/cyface/persistence/serialization/AttachmentSerializer.kt index 0f2f852b..4f832cd1 100644 --- a/persistence/src/main/kotlin/de/cyface/persistence/serialization/AttachmentSerializer.kt +++ b/persistence/src/main/kotlin/de/cyface/persistence/serialization/AttachmentSerializer.kt @@ -28,8 +28,6 @@ import java.nio.file.Files * Serializes a [Attachment] in the [MeasurementSerializer.TRANSFER_FILE_FORMAT_VERSION]. * * @author Armin Schnabel - * @version 1.0.0 - * @since 7.10.0 */ class AttachmentSerializer { /** diff --git a/persistence/src/main/kotlin/de/cyface/persistence/serialization/MeasurementSerializer.kt b/persistence/src/main/kotlin/de/cyface/persistence/serialization/MeasurementSerializer.kt index cc9f4d02..94be4124 100644 --- a/persistence/src/main/kotlin/de/cyface/persistence/serialization/MeasurementSerializer.kt +++ b/persistence/src/main/kotlin/de/cyface/persistence/serialization/MeasurementSerializer.kt @@ -185,6 +185,11 @@ class MeasurementSerializer { * layer serialized in the [MeasurementSerializer.TRANSFER_FILE_FORMAT_VERSION] format, ready to be * transferred. * + * Attention: + * We don't wrap the attachments in the `cyf` wrapper, as: + * - Most our project currently prefer the plain JPG, CSV, ZIP, etc. formats + * - We have a version in meta data, and currently have version 1 for attachment files format. + * * No compression is used as we're mostly transferring JPG files right now which are pre-compressed. * * @param fileOutputStream the `FileInputStream` to write the compressed data to diff --git a/persistence/src/main/kotlin/de/cyface/persistence/serialization/TransferFileSerializer.kt b/persistence/src/main/kotlin/de/cyface/persistence/serialization/TransferFileSerializer.kt index 3391fe59..fd474e3b 100644 --- a/persistence/src/main/kotlin/de/cyface/persistence/serialization/TransferFileSerializer.kt +++ b/persistence/src/main/kotlin/de/cyface/persistence/serialization/TransferFileSerializer.kt @@ -29,7 +29,6 @@ import de.cyface.persistence.content.AbstractCyfaceTable.Companion.DATABASE_QUER import de.cyface.persistence.model.Attachment import de.cyface.persistence.model.Measurement import de.cyface.protos.model.Event -import de.cyface.protos.model.File.FileType import de.cyface.protos.model.LocationRecords import de.cyface.protos.model.MeasurementBytes import de.cyface.serializer.DataSerializable @@ -45,8 +44,6 @@ import java.io.IOException * Serializes [MeasurementSerializer.TRANSFER_FILE_FORMAT_VERSION] files. * * @author Armin Schnabel - * @version 3.1.0 - * @since 5.0.0 */ object TransferFileSerializer { /** @@ -253,13 +250,16 @@ object TransferFileSerializer { * Implements the core algorithm of loading data of a [Attachment] from the [PersistenceLayer] * and serializing it into an array of bytes, ready to be transferred. * - * We use the {@param loader} to access the measurement data. FIXME? - * * We assemble the data using a buffer to avoid OOM exceptions. * * **ATTENTION:** The caller must make sure the {@param bufferedOutputStream} is closed when no longer needed * or the app crashes. * + * Attention: + * We don't wrap the attachments in the `cyf` wrapper, as: + * - Most our project currently prefer the plain JPG, CSV, ZIP, etc. formats + * - We have a version in meta data, and currently have version 1 for attachment files format. + * * @param bufferedOutputStream The `OutputStream` to which the serialized data should be written. Injecting * this allows us to compress the serialized data without the need to write it into a temporary file. * We require a [BufferedOutputStream] for performance reasons. @@ -274,7 +274,7 @@ object TransferFileSerializer { ) { val attachment = loadAttachment(reference) - val builder = de.cyface.protos.model.Measurement.newBuilder() + /*val builder = de.cyface.protos.model.Measurement.newBuilder() .setFormatVersion(MeasurementSerializer.TRANSFER_FILE_FORMAT_VERSION.toInt()) when (reference.type) { // TODO: zip all attachments @@ -289,18 +289,19 @@ object TransferFileSerializer { else -> { throw IllegalArgumentException("Unsupported type: ${reference.type}") } - } + }*/ // Currently loading one image per transfer file into memory (~ 2-5 MB / image). // - To load add all high-res image data or video data in the future we cannot use the pre-compiled // builder but have to stream the data without loading it into memory to avoid an OOM exception. val transferFileHeader = DataSerializable.transferFileHeader() - val measurementBytes = builder.build().toByteArray() + //val uploadBytes = builder.build().toByteArray() + val uploadBytes = attachment.bytes.toByteArray() try { // The stream must be closed by the caller in a finally catch withContext(Dispatchers.IO) { bufferedOutputStream.write(transferFileHeader) - bufferedOutputStream.write(measurementBytes) + bufferedOutputStream.write(uploadBytes) bufferedOutputStream.flush() } } catch (e: IOException) { @@ -310,7 +311,7 @@ object TransferFileSerializer { TAG, String.format( "Serialized attachment: %s", DataSerializable.humanReadableSize( - (transferFileHeader.size + measurementBytes.size).toLong(), + (transferFileHeader.size + uploadBytes.size).toLong(), true ) ) diff --git a/synchronization/src/main/kotlin/de/cyface/synchronization/SyncAdapter.kt b/synchronization/src/main/kotlin/de/cyface/synchronization/SyncAdapter.kt index 57c30669..75b688b9 100644 --- a/synchronization/src/main/kotlin/de/cyface/synchronization/SyncAdapter.kt +++ b/synchronization/src/main/kotlin/de/cyface/synchronization/SyncAdapter.kt @@ -37,7 +37,6 @@ import de.cyface.persistence.exception.NoSuchMeasurementException import de.cyface.persistence.model.AttachmentStatus import de.cyface.persistence.model.Measurement import de.cyface.persistence.model.MeasurementStatus -import de.cyface.persistence.model.ParcelableGeoLocation import de.cyface.persistence.serialization.MeasurementSerializer import de.cyface.protos.model.File.FileType import de.cyface.synchronization.ErrorHandler.ErrorCode @@ -301,6 +300,7 @@ class SyncAdapter private constructor( if (isSyncRequestAborted(account, authority)) return + @Suppress("SpellCheckingInspection") val indexWithinMeasurement = 1 + syncedAttachments + attachmentIndex // ccyf is index 0 val progressListener = DefaultUploadProgressListener( measurementCount, @@ -313,6 +313,7 @@ class SyncAdapter private constructor( val attachmentMeta = attachmentMeta(measurementMeta, attachment.id) error = !syncAttachment( attachmentMeta, + attachment.type, syncPerformer, transferTempFile, syncResult, @@ -433,6 +434,7 @@ class SyncAdapter private constructor( private suspend fun syncAttachment( attachment: Attachment, + attachmentType: FileType, syncPerformer: SyncPerformer, transferFile: File?, syncResult: SyncResult, @@ -456,7 +458,7 @@ class SyncAdapter private constructor( } else { val attachmentId = attachment.identifier.attachmentIdentifier val fileName = - "${attachment.identifier.deviceIdentifier}_${attachment.identifier.measurementIdentifier}_$attachmentId.${TRANSFER_FILE_EXTENSION}" + "${attachment.identifier.deviceIdentifier}_${attachment.identifier.measurementIdentifier}_$attachmentId.${attachmentType.name.lowercase()}" val result = syncPerformer.sendData( uploader, syncResult, @@ -745,11 +747,6 @@ class SyncAdapter private constructor( */ const val MOCK_IS_CONNECTED_TO_RETURN_TRUE = "mocked_periodic_sync_check_false" - /** - * The file extension of the attachment file which is transmitted on synchronization. - */ - private const val TRANSFER_FILE_EXTENSION = "cyf" - /** * The file extension of the measurement file which is transmitted on synchronization. */ From 62c8cfc5981cd3f6d332f30e1d2e83b9432dd674 Mon Sep 17 00:00:00 2001 From: Armin Date: Wed, 23 Oct 2024 11:14:55 +0100 Subject: [PATCH 3/5] [RFR-1179] Add documentation --- .../serialization/TransferFileSerializer.kt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/persistence/src/main/kotlin/de/cyface/persistence/serialization/TransferFileSerializer.kt b/persistence/src/main/kotlin/de/cyface/persistence/serialization/TransferFileSerializer.kt index fd474e3b..4ddf4d64 100644 --- a/persistence/src/main/kotlin/de/cyface/persistence/serialization/TransferFileSerializer.kt +++ b/persistence/src/main/kotlin/de/cyface/persistence/serialization/TransferFileSerializer.kt @@ -39,6 +39,7 @@ import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext import java.io.BufferedOutputStream import java.io.IOException +import java.util.Locale /** * Serializes [MeasurementSerializer.TRANSFER_FILE_FORMAT_VERSION] files. @@ -274,10 +275,17 @@ object TransferFileSerializer { ) { val attachment = loadAttachment(reference) + // In case we switch back to the cyf wrapper for attachments, we need to adjust the code: + // Out Protobuf format only supports one `capturing_log` file, but we collect multiple + // files which do not fit the "images" or "video" categories (i.e. CSV, JSON). If we would + // upload all attachments in one request, we would need to compress them into one ZIP files. + // That is why we added the file format "ZIP" to the Protobuf message definition. + // But as we upload each attachment separately, even with the cyf wrapper we should be fine + // with one `capturing_log` support, as we can just add this one file as such. + // So if you enable the cyf wrapping code below again, make sure all log files are uploaded. /*val builder = de.cyface.protos.model.Measurement.newBuilder() .setFormatVersion(MeasurementSerializer.TRANSFER_FILE_FORMAT_VERSION.toInt()) when (reference.type) { - // TODO: zip all attachments FileType.JSON, FileType.CSV -> { builder.capturingLog = attachment } @@ -309,6 +317,7 @@ object TransferFileSerializer { } Log.d( TAG, String.format( + Locale.getDefault(), "Serialized attachment: %s", DataSerializable.humanReadableSize( (transferFileHeader.size + uploadBytes.size).toLong(), From fb5da2ad8db826fb81c4100680514b422642ee75 Mon Sep 17 00:00:00 2001 From: Armin Date: Wed, 23 Oct 2024 11:54:57 +0100 Subject: [PATCH 4/5] Upgrade uploader to 1.5.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 58f6c78b..a9e9c3b9 100644 --- a/build.gradle +++ b/build.gradle @@ -53,7 +53,7 @@ ext { // When `android-app` is checkout out, `rootProject.uploaderVersion` in the submodules will point // to `android-app/build.gradle` instead of `submodule/build.gradle`. cyfaceSerializationVersion = "3.4.2" - cyfaceUploaderVersion = "1.5.0-alpha7" // FIXME + cyfaceUploaderVersion = "1.5.0" // Android SDK versions minSdkVersion = 21 // device support From 68d494dfc76dc55643fbb63427c9c734afee7d39 Mon Sep 17 00:00:00 2001 From: Armin Date: Wed, 23 Oct 2024 12:04:10 +0100 Subject: [PATCH 5/5] Fix test build --- .../kotlin/de/cyface/synchronization/MockedUploader.kt | 5 +++-- .../kotlin/de/cyface/synchronization/SyncPerformerTest.kt | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/synchronization/src/androidTest/kotlin/de/cyface/synchronization/MockedUploader.kt b/synchronization/src/androidTest/kotlin/de/cyface/synchronization/MockedUploader.kt index 5abcca4e..63d241dd 100644 --- a/synchronization/src/androidTest/kotlin/de/cyface/synchronization/MockedUploader.kt +++ b/synchronization/src/androidTest/kotlin/de/cyface/synchronization/MockedUploader.kt @@ -23,6 +23,7 @@ import de.cyface.uploader.UploadProgressListener import de.cyface.uploader.Uploader import de.cyface.uploader.model.Attachment import de.cyface.uploader.model.Measurement +import de.cyface.uploader.model.Uploadable import java.io.File import java.net.MalformedURLException import java.net.URL @@ -34,7 +35,7 @@ import java.net.URL */ internal class MockedUploader : Uploader { - override fun measurementsEndpoint(): URL { + override fun measurementsEndpoint(uploadable: Uploadable): URL { return try { URL("https://mocked.cyface.de/api/v123/measurements") } catch (e: MalformedURLException) { @@ -42,7 +43,7 @@ internal class MockedUploader : Uploader { } } - override fun attachmentsEndpoint(deviceId: String, measurementId: Long): URL { + override fun attachmentsEndpoint(uploadable: Uploadable): URL { return try { URL("https://mocked.cyface.de/api/v123/measurements/did/mid/attachments") } catch (e: MalformedURLException) { diff --git a/synchronization/src/androidTest/kotlin/de/cyface/synchronization/SyncPerformerTest.kt b/synchronization/src/androidTest/kotlin/de/cyface/synchronization/SyncPerformerTest.kt index 1b62eba8..60ae1dee 100644 --- a/synchronization/src/androidTest/kotlin/de/cyface/synchronization/SyncPerformerTest.kt +++ b/synchronization/src/androidTest/kotlin/de/cyface/synchronization/SyncPerformerTest.kt @@ -57,6 +57,7 @@ import de.cyface.uploader.exception.UploadFailed import de.cyface.uploader.exception.UploadSessionExpired import de.cyface.uploader.model.Attachment import de.cyface.uploader.model.MeasurementIdentifier +import de.cyface.uploader.model.Uploadable import de.cyface.uploader.model.metadata.ApplicationMetaData import de.cyface.uploader.model.metadata.AttachmentMetaData import de.cyface.uploader.model.metadata.DeviceMetaData @@ -309,12 +310,12 @@ class SyncPerformerTest { // Mock the actual post request val mockedUploader = object : Uploader { - override fun measurementsEndpoint(): URL { + override fun measurementsEndpoint(uploadable: Uploadable): URL { return URL("https://mocked.cyface.de/api/v123/measurements") } - override fun attachmentsEndpoint(deviceId: String, measurementId: Long): URL { - return URL("https://mocked.cyface.de/api/v123/measurements/$deviceId/$measurementId/attachments") + override fun attachmentsEndpoint(uploadable: Uploadable): URL { + return URL("https://mocked.cyface.de/api/v123/measurements/${uploadable.deviceId()}/${uploadable.measurementId()}/attachments") } override fun uploadMeasurement(