[LEIP-164] Support attachments (#289)

* Fix typo

* Support NIO through desugaring

* Rename Default-/FileDao to Default-/FileIOHandler

* [LEIP-174] Add classes for the File table

* Add database migration and set version to 19

* Add update method to FileDao

* Add MeasurementStatus UPLOADING

* Refactor onPerformSync to lower complexity

* Rename UPLOADING state to a more precise SYNCABLE_ATTACHMENTS

* Enable NIO in desugaring in remaining modules

* [LEIP-176] Upload attachments

* Add locationTimestamp to File table

* [LEIP-179] Wrap attachments as cyf

* [LEIP-182] Add file counts to meta data

* Fix upload progress when continuing with later attachments

* Convert CapturingPersistenceBehaviour to kotlin

* [LEIP-186] Mark database operations as suspend

* [LEIP-184] Add filesSize to metadata

* [LEIP-185] Fix resource failed to call end

* Add FileDaoTest

* Fix tests after changing dao functions to suspend

* Convert ProtoTest to kotlin

* Add ProtoTest.testWithFileAttachments

* Fix SyncPerformerTest

* Add SyncPerformerTest.testSendData_withAttachments

* [LEIP-181] Add SyncAdapterTest.testOnPerformSync with/out existing attachments

* Upgrade Cyface dependencies to beta1

* Upload the attachments to files endpoint

* Fix tests

* Ignore false-positive lint errors (we use desugaring for NIO)

* Update code version

* Upgrade serialization and uploader to final version

* Fix typo

* Adjust to renamed uploader interface

* Rename measurement.file to .attachment in persistence

* Rename measurement.files to .attachments

build.gradle

@@ -21,7 +21,7 @@
  *
  * @author Armin Schnabel
  * @author Klemens Muthmann
- * @version 2.10.0
+ * @version 2.11.0
  * @since 1.0.0
  */
 
@@ -51,8 +51,8 @@ ext {
 
     // Cyface dependencies
     cyfaceUtilsVersion = "4.0.4"
-    cyfaceSerializationVersion = "3.0.0"
-    cyfaceUploaderVersion = "1.0.0"
+    cyfaceSerializationVersion = "3.2.0"
+    cyfaceUploaderVersion = "1.2.0"
 
     // Android SDK versions
     minSdkVersion = 21 // device support
@@ -61,6 +61,7 @@ ext {
     buildToolsVersion = '34.0.0' // optional, if defined, use latest (SDK Manager > SDK Tools)
     // 1.1.0-alpha05: ':persistence:mergeExtDexDebugAndroidTest' fails (transitive dependency)
     datastoreVersion = "1.1.0-alpha04" // only 1.1.0 supports multi-process datastore
+    desugaringVersion = "2.0.3"
 
     // Android dependencies
     androidxAnnotationVersion = '1.6.0'

datacapturing/build.gradle

@@ -21,7 +21,7 @@
  *
  * @author Armin Schnabel
  * @author Klemens Muthmann
- * @version 3.3.0
+ * @version 3.4.0
  * @since 1.0.0
  */
 
@@ -95,8 +95,10 @@ android {
         }
     }
 
-    // Enabling desugaring to support Java 8 and Java 11 features
     compileOptions {
+        // Enabling desugaring to support Java 8 and Java 11 features
+        coreLibraryDesugaringEnabled true
+        // Set Java compatibility
         sourceCompatibility rootProject.ext.sourceCompatibility
         targetCompatibility rootProject.ext.targetCompatibility
     }
@@ -120,6 +122,8 @@ dependencies {
     implementation "androidx.annotation:annotation:$rootProject.ext.androidxAnnotationVersion"
     implementation "androidx.appcompat:appcompat:$rootProject.ext.androidxAppCompatVersion"
     implementation "androidx.localbroadcastmanager:localbroadcastmanager:$rootProject.ext.localbroadcastmanagerVersion"
+    // Add support desugaring with for NIO
+    coreLibraryDesugaring "com.android.tools:desugar_jdk_libs_nio:$rootProject.ext.desugaringVersion"
 
     // `Room` API used to access database
     // Further room libraries and Kotlin support see Room guide.

datacapturing/src/androidTest/kotlin/de/cyface/datacapturing/model/CapturedDataWriterTest.kt

@@ -28,9 +28,9 @@ import de.cyface.datacapturing.persistence.WritingDataCompletedCallback
 import de.cyface.persistence.DefaultPersistenceLayer
 import de.cyface.persistence.PersistenceBehaviour
 import de.cyface.persistence.PersistenceLayer
-import de.cyface.persistence.dao.DefaultFileDao
-import de.cyface.persistence.dao.FileDao
 import de.cyface.persistence.exception.NoSuchMeasurementException
+import de.cyface.persistence.io.DefaultFileIOHandler
+import de.cyface.persistence.io.FileIOHandler
 import de.cyface.persistence.model.EventType
 import de.cyface.persistence.model.GeoLocation
 import de.cyface.persistence.model.MeasurementStatus
@@ -63,7 +63,7 @@ import java.util.concurrent.locks.ReentrantLock
  *
  * @author Klemens Muthmann
  * @author Armin Schnabel
- * @version 5.6.4
+ * @version 5.6.5
  * @since 1.0.0
  */
 @RunWith(AndroidJUnit4::class)
@@ -164,7 +164,7 @@ class CapturedDataWriterTest {
      */
     @Test
     @Throws(NoSuchFileException::class, InvalidProtocolBufferException::class)
-    fun testStoreData() {
+    fun testStoreData() = runBlocking {
         // Manually trigger data capturing (new measurement with sensor data and a location)
         val (id) = oocut!!.newMeasurement(Modality.UNKNOWN)
         val lock: Lock = ReentrantLock()
@@ -186,6 +186,7 @@ class CapturedDataWriterTest {
         )
         lock.lock()
         try {
+            @Suppress("BlockingMethodInNonBlockingContext")
             condition.await(2, TimeUnit.SECONDS)
         } catch (e: InterruptedException) {
             throw IllegalStateException(e)
@@ -195,17 +196,17 @@ class CapturedDataWriterTest {
         capturingBehaviour!!.storeLocation(testLocation(1L), id)
 
         // Check if the captured data was persisted
-        val fileDao: FileDao = DefaultFileDao()
+        val fileIOHandler: FileIOHandler = DefaultFileIOHandler()
         val locations = oocut!!.locationDao!!.getAll()
         MatcherAssert.assertThat(locations.size, Is.`is`(IsEqual.equalTo(TEST_LOCATION_COUNT)))
 
         // Point3Ds
-        val accelerationsFile = loadFile(context!!, fileDao, id, Point3DType.ACCELERATION)
-        val rotationsFile = loadFile(context!!, fileDao, id, Point3DType.ROTATION)
-        val directionsFile = loadFile(context!!, fileDao, id, Point3DType.DIRECTION)
-        val accelerations = deserialize(fileDao, accelerationsFile.file, Point3DType.ACCELERATION)
-        val rotations = deserialize(fileDao, rotationsFile.file, Point3DType.ROTATION)
-        val directions = deserialize(fileDao, directionsFile.file, Point3DType.DIRECTION)
+        val accelerationsFile = loadFile(context!!, fileIOHandler, id, Point3DType.ACCELERATION)
+        val rotationsFile = loadFile(context!!, fileIOHandler, id, Point3DType.ROTATION)
+        val directionsFile = loadFile(context!!, fileIOHandler, id, Point3DType.DIRECTION)
+        val accelerations = deserialize(fileIOHandler, accelerationsFile.file, Point3DType.ACCELERATION)
+        val rotations = deserialize(fileIOHandler, rotationsFile.file, Point3DType.ROTATION)
+        val directions = deserialize(fileIOHandler, directionsFile.file, Point3DType.DIRECTION)
         val accelerationBatch = accelerations.accelerationsBinary.getAccelerations(0)
         MatcherAssert.assertThat(
             accelerationBatch.timestampCount, Is.`is`(
@@ -302,15 +303,15 @@ class CapturedDataWriterTest {
             ) // because we don't clean it up currently
 
             // Make sure nothing is left of the Point3DFiles
-            val accelerationsFolder = oocut!!.fileDao.getFolderPath(
+            val accelerationsFolder = oocut!!.fileIOHandler.getFolderPath(
                 context!!,
                 Point3DFile.ACCELERATIONS_FOLDER_NAME
             )
-            val rotationsFolder = oocut!!.fileDao.getFolderPath(
+            val rotationsFolder = oocut!!.fileIOHandler.getFolderPath(
                 context!!,
                 Point3DFile.ROTATIONS_FOLDER_NAME
             )
-            val directionsFolder = oocut!!.fileDao.getFolderPath(
+            val directionsFolder = oocut!!.fileIOHandler.getFolderPath(
                 context!!,
                 Point3DFile.DIRECTIONS_FOLDER_NAME
             )
@@ -339,7 +340,7 @@ class CapturedDataWriterTest {
      * Tests whether deleting a measurement actually remove that measurement together with all corresponding data.
      */
     @Test
-    fun testDeleteMeasurement() {
+    fun testDeleteMeasurement() = runBlocking {
 
         // Arrange
         val measurement = oocut!!.newMeasurement(Modality.UNKNOWN)
@@ -358,6 +359,7 @@ class CapturedDataWriterTest {
         capturingBehaviour!!.storeData(testData(), measurementId, callback)
         lock.lock()
         try {
+            @Suppress("BlockingMethodInNonBlockingContext")
             condition.await(2, TimeUnit.SECONDS)
         } catch (e: InterruptedException) {
             throw IllegalStateException(e)
@@ -371,15 +373,15 @@ class CapturedDataWriterTest {
         oocut!!.delete(measurement.id)
 
         // Assert
-        val accelerationFile = oocut!!.fileDao.getFilePath(
+        val accelerationFile = oocut!!.fileIOHandler.getFilePath(
             context!!, measurementId,
             Point3DFile.ACCELERATIONS_FOLDER_NAME, Point3DFile.ACCELERATIONS_FILE_EXTENSION
         )
-        val rotationFile = oocut!!.fileDao.getFilePath(
+        val rotationFile = oocut!!.fileIOHandler.getFilePath(
             context!!, measurementId,
             Point3DFile.ROTATIONS_FOLDER_NAME, Point3DFile.ROTATION_FILE_EXTENSION
         )
-        val directionFile = oocut!!.fileDao.getFilePath(
+        val directionFile = oocut!!.fileIOHandler.getFilePath(
             context!!, measurementId,
             Point3DFile.DIRECTIONS_FOLDER_NAME, Point3DFile.DIRECTION_FILE_EXTENSION
         )

datacapturing/src/androidTest/kotlin/de/cyface/datacapturing/persistence/PersistenceLayerTest.kt

@@ -44,7 +44,7 @@ import org.junit.runner.RunWith
  *
  * @author Klemens Muthmann
  * @author Armin Schnabel
- * @version 1.5.7
+ * @version 1.5.8
  * @since 2.0.3
  */
 @RunWith(AndroidJUnit4::class)
@@ -100,7 +100,7 @@ class PersistenceLayerTest {
         assertThat(oocut!!.hasMeasurement(MeasurementStatus.OPEN), equalTo(false))
         oocut!!.newMeasurement(Modality.UNKNOWN)
         assertThat(oocut!!.hasMeasurement(MeasurementStatus.OPEN), equalTo(true))
-        assertThat(oocut!!.loadMeasurements(MeasurementStatus.FINISHED)?.size, equalTo(1))
+        assertThat(oocut!!.loadMeasurements(MeasurementStatus.FINISHED).size, equalTo(1))
     }
 
     /**
@@ -110,7 +110,7 @@ class PersistenceLayerTest {
      */
     @Test
     fun testLoadFinishedMeasurements_noMeasurements() {
-        assertThat(oocut!!.loadMeasurements(MeasurementStatus.FINISHED)?.isEmpty(), equalTo(true))
+        assertThat(oocut!!.loadMeasurements(MeasurementStatus.FINISHED).isEmpty(), equalTo(true))
     }
 
     /**
@@ -134,11 +134,11 @@ class PersistenceLayerTest {
         capturingBehaviour!!.updateRecentMeasurement(MeasurementStatus.FINISHED)
         val finishedMeasurements = oocut!!.loadMeasurements(MeasurementStatus.FINISHED)
         assertThat(
-            finishedMeasurements!!.size,
+            finishedMeasurements.size,
             CoreMatchers.`is`(equalTo(1))
         )
         assertThat(
-            finishedMeasurements[0]!!.id,
+            finishedMeasurements[0].id,
             CoreMatchers.`is`(equalTo(measurement.id))
         )
     }
@@ -158,29 +158,29 @@ class PersistenceLayerTest {
         // Check that measurement was marked as synced
         val syncedMeasurements = oocut!!.loadMeasurements(MeasurementStatus.SYNCED)
         assertThat(
-            syncedMeasurements!!.size,
+            syncedMeasurements.size,
             CoreMatchers.`is`(equalTo(1))
         )
         assertThat(
-            syncedMeasurements[0]!!.id, CoreMatchers.`is`(
+            syncedMeasurements[0].id, CoreMatchers.`is`(
                 equalTo(
                     id
                 )
             )
         )
 
         // Check that sensor data was deleted
-        val accelerationFile = oocut!!.fileDao.getFilePath(
+        val accelerationFile = oocut!!.fileIOHandler.getFilePath(
             context!!, id,
             Point3DFile.ACCELERATIONS_FOLDER_NAME, Point3DFile.ACCELERATIONS_FILE_EXTENSION
         )
         Validate.isTrue(!accelerationFile.exists())
-        val rotationFile = oocut!!.fileDao.getFilePath(
+        val rotationFile = oocut!!.fileIOHandler.getFilePath(
             context!!, id,
             Point3DFile.ROTATIONS_FOLDER_NAME, Point3DFile.ROTATION_FILE_EXTENSION
         )
         Validate.isTrue(!rotationFile.exists())
-        val directionFile = oocut!!.fileDao.getFilePath(
+        val directionFile = oocut!!.fileIOHandler.getFilePath(
             context!!, id,
             Point3DFile.DIRECTIONS_FOLDER_NAME, Point3DFile.DIRECTION_FILE_EXTENSION
         )
@@ -194,7 +194,7 @@ class PersistenceLayerTest {
      */
     @Test
     @Throws(NoSuchMeasurementException::class)
-    fun testGetSyncableMeasurement() {
+    fun testGetSyncableMeasurement() = runBlocking {
 
         // Create a synchronized measurement
         insertSampleMeasurementWithData(context!!, MeasurementStatus.SYNCED, oocut!!, 1, 1)
@@ -210,9 +210,9 @@ class PersistenceLayerTest {
 
         // Check that syncable measurements = finishedMeasurement
         val loadedMeasurements = oocut!!.loadMeasurements(MeasurementStatus.FINISHED)
-        assertThat(loadedMeasurements!!.size, CoreMatchers.`is`(1))
+        assertThat(loadedMeasurements.size, CoreMatchers.`is`(1))
         assertThat(
-            loadedMeasurements[0]!!.id, CoreMatchers.`is`(
+            loadedMeasurements[0].id, CoreMatchers.`is`(
                 equalTo(
                     id
                 )