From 0ad404fb3b9fe3df5767be5b34666f789797bd55 Mon Sep 17 00:00:00 2001 From: Woohyeok Choi Date: Thu, 6 Feb 2020 23:22:02 +0900 Subject: [PATCH] Fix synchronization --- app/build.gradle | 4 +- app/objectbox-models/default.json | 40 ++--- .../main/kotlin/kaist/iclab/abclogger/App.kt | 2 + .../kotlin/kaist/iclab/abclogger/Debug.kt | 11 +- .../kotlin/kaist/iclab/abclogger/ObjBox.kt | 20 ++- .../kaist/iclab/abclogger/SyncWorker.kt | 140 +++++++++--------- .../kaist/iclab/abclogger/collector/Base.kt | 2 +- .../externalsensor/polar/PolarH10Collector.kt | 2 +- .../collector/keylog/KeyLogCollector.kt | 2 +- 9 files changed, 125 insertions(+), 98 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 1498e2e..387de28 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -28,8 +28,8 @@ android { applicationId 'kaist.iclab.abclogger' minSdkVersion 23 targetSdkVersion 29 - versionCode 16 - versionName "0.9.6-f" + versionCode 17 + versionName "0.9.7" setProperty('archivesBaseName', applicationId + "-v" + versionName) buildConfigField("String", "DB_NAME", "\"abc-logger\"") buildConfigField("String", "PREF_NAME", "\"abc-logger-pref\"") diff --git a/app/objectbox-models/default.json b/app/objectbox-models/default.json index 91ee8c4..7723d0c 100644 --- a/app/objectbox-models/default.json +++ b/app/objectbox-models/default.json @@ -30,7 +30,6 @@ }, { "id": "7:3036374616977976360", - "indexId": "1:158062895926330245", "name": "isUploaded" }, { @@ -83,7 +82,6 @@ }, { "id": "7:9166426228699522215", - "indexId": "2:256201055211819376", "name": "isUploaded" }, { @@ -144,7 +142,6 @@ }, { "id": "7:5483891281740835030", - "indexId": "3:6415803526992157091", "name": "isUploaded" }, { @@ -189,7 +186,6 @@ }, { "id": "7:8554306890949689513", - "indexId": "4:6115229358585485744", "name": "isUploaded" }, { @@ -254,7 +250,6 @@ }, { "id": "7:3037623066580555738", - "indexId": "5:5988685338798823253", "name": "isUploaded" }, { @@ -311,7 +306,6 @@ }, { "id": "7:6227451213385940735", - "indexId": "6:3657864108008906612", "name": "isUploaded" }, { @@ -348,7 +342,6 @@ }, { "id": "7:859617273231228374", - "indexId": "7:2042024887155835266", "name": "isUploaded" }, { @@ -405,7 +398,6 @@ }, { "id": "7:950647487466205370", - "indexId": "8:8326708012163642480", "name": "isUploaded" }, { @@ -458,7 +450,6 @@ }, { "id": "7:8956753562347954950", - "indexId": "9:9155900179125864887", "name": "isUploaded" }, { @@ -495,7 +486,6 @@ }, { "id": "7:2244412886878113093", - "indexId": "10:539979283176053442", "name": "isUploaded" }, { @@ -552,7 +542,6 @@ }, { "id": "7:6938767616159991922", - "indexId": "11:6497556530959105106", "name": "isUploaded" }, { @@ -629,7 +618,6 @@ }, { "id": "7:4273578758557398982", - "indexId": "12:6039103646252155121", "name": "isUploaded" }, { @@ -670,7 +658,6 @@ }, { "id": "7:4541625880572395911", - "indexId": "13:5537872971623978248", "name": "isUploaded" }, { @@ -711,7 +698,6 @@ }, { "id": "7:6025001404903907859", - "indexId": "16:7250749398665699724", "name": "isUploaded" }, { @@ -768,7 +754,6 @@ }, { "id": "7:929324376506956077", - "indexId": "17:6738167932301395690", "name": "isUploaded" }, { @@ -833,7 +818,6 @@ }, { "id": "7:3315443384237767477", - "indexId": "19:5958550830631192722", "name": "isUploaded" }, { @@ -882,7 +866,6 @@ }, { "id": "7:8478344564654657002", - "indexId": "21:3208699785279838195", "name": "isUploaded" }, { @@ -959,7 +942,6 @@ }, { "id": "7:4235451395354468046", - "indexId": "22:7728963676690170246", "name": "isUploaded" }, { @@ -1020,7 +1002,6 @@ }, { "id": "6:6584685521572670325", - "indexId": "23:8572453880263911372", "name": "isUploaded" }, { @@ -1059,7 +1040,26 @@ 3186997738512980675, 6303027327249555250, 7327424447267581296, - 3064297410742851827 + 3064297410742851827, + 3208699785279838195, + 8572453880263911372, + 8326708012163642480, + 5958550830631192722, + 539979283176053442, + 256201055211819376, + 6039103646252155121, + 6415803526992157091, + 6497556530959105106, + 7728963676690170246, + 3657864108008906612, + 2042024887155835266, + 6115229358585485744, + 158062895926330245, + 5988685338798823253, + 9155900179125864887, + 5537872971623978248, + 7250749398665699724, + 6738167932301395690 ], "retiredPropertyUids": [ 1683179073586285052, diff --git a/app/src/main/kotlin/kaist/iclab/abclogger/App.kt b/app/src/main/kotlin/kaist/iclab/abclogger/App.kt index 89b6bb6..ddc148a 100644 --- a/app/src/main/kotlin/kaist/iclab/abclogger/App.kt +++ b/app/src/main/kotlin/kaist/iclab/abclogger/App.kt @@ -23,6 +23,8 @@ class App : Application(){ } GlobalScope.launch { ABC.bind(this@App) + + // Debug.generateEntities(50000) } } diff --git a/app/src/main/kotlin/kaist/iclab/abclogger/Debug.kt b/app/src/main/kotlin/kaist/iclab/abclogger/Debug.kt index 926ab0c..5cbf195 100644 --- a/app/src/main/kotlin/kaist/iclab/abclogger/Debug.kt +++ b/app/src/main/kotlin/kaist/iclab/abclogger/Debug.kt @@ -1,6 +1,8 @@ package kaist.iclab.abclogger import android.util.Log +import kaist.iclab.abclogger.collector.activity.PhysicalActivityTransitionEntity +import kaist.iclab.abclogger.collector.fill import kaist.iclab.abclogger.collector.survey.SurveyEntity import java.util.concurrent.TimeUnit @@ -163,6 +165,13 @@ object Debug { } })*/ + suspend fun generateEntities(size: Int) { + (0..size).map { + PhysicalActivityTransitionEntity(type = "ASDFASDFASDFASDFASDFASDFASDF", isEntered = true).fill(System.currentTimeMillis()) + }.let { ObjBox.put(it) } + Log.d("ZXCV", "${ObjBox.boxFor()?.count()}") + } + suspend fun generateSurveyEntities (size: Int) { val time = System.currentTimeMillis() (0..size).map { @@ -173,7 +182,7 @@ object Debug { timeoutPolicy = "DISABLED", deliveredTime = time + it * TimeUnit.MINUTES.toMillis(5), json = testJson - ) + ).fill(System.currentTimeMillis()) }.let { ObjBox.put(it) } val query = ObjBox.boxFor()?.query()?.build() ?: return diff --git a/app/src/main/kotlin/kaist/iclab/abclogger/ObjBox.kt b/app/src/main/kotlin/kaist/iclab/abclogger/ObjBox.kt index 4c31dff..afa21d5 100644 --- a/app/src/main/kotlin/kaist/iclab/abclogger/ObjBox.kt +++ b/app/src/main/kotlin/kaist/iclab/abclogger/ObjBox.kt @@ -4,6 +4,7 @@ import android.content.Context import androidx.core.app.NotificationManagerCompat import io.objectbox.BoxStore import io.objectbox.kotlin.boxFor +import kaist.iclab.abclogger.collector.Base import kaist.iclab.abclogger.collector.MyObjectBox import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -90,11 +91,11 @@ object ObjBox { fun maxSizeInBytes() = Prefs.maxDbSize * 1000L - inline fun boxFor() = try { boxStore.get()?.boxFor() } catch (e: Exception) { null } + inline fun boxFor() = try { boxStore.get()?.boxFor() } catch (e: Exception) { null } - fun boxFor(clazz: Class) = try { boxStore.get()?.boxFor(clazz) } catch (e: Exception) { null } + fun boxFor(clazz: Class) = try { boxStore.get()?.boxFor(clazz) } catch (e: Exception) { null } - inline fun putSync(entity: T?): Long { + inline fun putSync(entity: T?): Long { entity ?: return -1L if (boxStore.get()?.isClosed != false) return -1 @@ -102,7 +103,7 @@ object ObjBox { return try { boxFor()?.put(entity) } catch (e: Exception) { null } ?: -1 } - inline fun putSync(entities: Collection?) { + inline fun putSync(entities: Collection?) { if (entities.isNullOrEmpty()) return if (boxStore.get()?.isClosed != false) return @@ -110,7 +111,7 @@ object ObjBox { try { boxFor()?.put(entities) } catch (e: Exception) { } } - suspend inline fun put(entity: T?): Long = withContext(Dispatchers.IO) { + suspend inline fun put(entity: T?): Long = withContext(Dispatchers.IO) { entity ?: return@withContext -1 if (boxStore.get()?.isClosed != false) return@withContext -1 @@ -118,13 +119,20 @@ object ObjBox { return@withContext try { boxFor()?.put(entity) } catch (e: Exception) { null } ?: -1 } - suspend inline fun put(entities: Collection?) = withContext(Dispatchers.IO) { + suspend inline fun put(entities: Collection?) = withContext(Dispatchers.IO) { if (entities.isNullOrEmpty()) return@withContext if (boxStore.get()?.isClosed != false) return@withContext if (BuildConfig.DEBUG) AppLog.d(T::class.java.name, entities) try { boxFor()?.put(entities) } catch (e: Exception) { } } + + suspend inline fun removeByKeys(ids: Collection?) = withContext(Dispatchers.IO) { + if (ids.isNullOrEmpty()) return@withContext + if (boxStore.get()?.isClosed != false) return@withContext + + try { boxFor()?.removeByKeys(ids) } catch (e: Exception) { } + } } diff --git a/app/src/main/kotlin/kaist/iclab/abclogger/SyncWorker.kt b/app/src/main/kotlin/kaist/iclab/abclogger/SyncWorker.kt index 3f3fb11..6c13ac1 100644 --- a/app/src/main/kotlin/kaist/iclab/abclogger/SyncWorker.kt +++ b/app/src/main/kotlin/kaist/iclab/abclogger/SyncWorker.kt @@ -5,30 +5,47 @@ import android.util.Log import androidx.work.* import io.grpc.ManagedChannel import io.grpc.android.AndroidChannelBuilder -import io.objectbox.EntityInfo +import io.objectbox.Property import io.objectbox.query.Query import kaist.iclab.abclogger.collector.Base import kaist.iclab.abclogger.collector.activity.PhysicalActivityEntity +import kaist.iclab.abclogger.collector.activity.PhysicalActivityEntity_ import kaist.iclab.abclogger.collector.activity.PhysicalActivityTransitionEntity +import kaist.iclab.abclogger.collector.activity.PhysicalActivityTransitionEntity_ import kaist.iclab.abclogger.collector.appusage.AppUsageEventEntity +import kaist.iclab.abclogger.collector.appusage.AppUsageEventEntity_ import kaist.iclab.abclogger.collector.battery.BatteryEntity +import kaist.iclab.abclogger.collector.battery.BatteryEntity_ import kaist.iclab.abclogger.collector.bluetooth.BluetoothEntity +import kaist.iclab.abclogger.collector.bluetooth.BluetoothEntity_ import kaist.iclab.abclogger.collector.call.CallLogEntity +import kaist.iclab.abclogger.collector.call.CallLogEntity_ import kaist.iclab.abclogger.collector.event.DeviceEventEntity +import kaist.iclab.abclogger.collector.event.DeviceEventEntity_ import kaist.iclab.abclogger.collector.externalsensor.ExternalSensorEntity +import kaist.iclab.abclogger.collector.externalsensor.ExternalSensorEntity_ import kaist.iclab.abclogger.collector.install.InstalledAppEntity +import kaist.iclab.abclogger.collector.install.InstalledAppEntity_ import kaist.iclab.abclogger.collector.internalsensor.SensorEntity +import kaist.iclab.abclogger.collector.internalsensor.SensorEntity_ import kaist.iclab.abclogger.collector.keylog.KeyLogEntity +import kaist.iclab.abclogger.collector.keylog.KeyLogEntity_ import kaist.iclab.abclogger.collector.location.LocationEntity +import kaist.iclab.abclogger.collector.location.LocationEntity_ import kaist.iclab.abclogger.collector.media.MediaEntity +import kaist.iclab.abclogger.collector.media.MediaEntity_ import kaist.iclab.abclogger.collector.message.MessageEntity +import kaist.iclab.abclogger.collector.message.MessageEntity_ import kaist.iclab.abclogger.collector.notification.NotificationEntity +import kaist.iclab.abclogger.collector.notification.NotificationEntity_ import kaist.iclab.abclogger.collector.physicalstat.PhysicalStatEntity -import kaist.iclab.abclogger.collector.survey.Survey +import kaist.iclab.abclogger.collector.physicalstat.PhysicalStatEntity_ import kaist.iclab.abclogger.collector.survey.SurveyEntity import kaist.iclab.abclogger.collector.survey.SurveyEntity_ import kaist.iclab.abclogger.collector.traffic.DataTrafficEntity +import kaist.iclab.abclogger.collector.traffic.DataTrafficEntity_ import kaist.iclab.abclogger.collector.wifi.WifiEntity +import kaist.iclab.abclogger.collector.wifi.WifiEntity_ import kaist.iclab.abclogger.grpc.DataOperationsCoroutineGrpc import kaist.iclab.abclogger.grpc.DatumProto import kotlinx.coroutines.* @@ -48,7 +65,7 @@ class SyncWorker(context: Context, params: WorkerParameters) : CoroutineWorker(c ) ) - override suspend fun doWork(): Result { + override suspend fun doWork(): Result = withContext(Dispatchers.IO){ setForeground(foregroundInfo) val channel: ManagedChannel = AndroidChannelBuilder @@ -58,87 +75,78 @@ class SyncWorker(context: Context, params: WorkerParameters) : CoroutineWorker(c .executor(Dispatchers.IO.asExecutor()) .build() - val stub: DataOperationsCoroutineGrpc.DataOperationsCoroutineStub = DataOperationsCoroutineGrpc.newStubWithContext(channel) - .withDeadlineAfter(10, TimeUnit.SECONDS) + val stub = DataOperationsCoroutineGrpc.newStubWithContext(channel) - uploadAll(stub) - removeAll() + uploadAll(stub.withDeadlineAfter(5, TimeUnit.MINUTES)) Prefs.lastTimeDataSync = System.currentTimeMillis() - terminate(channel) + try { + channel.shutdownNow().awaitTermination(10, TimeUnit.SECONDS) + } catch (e: Exception) { + e.printStackTrace() + } - return Result.success() + return@withContext Result.success() } - private suspend fun uploadAll(stub: DataOperationsCoroutineGrpc.DataOperationsCoroutineStub) = withContext(Dispatchers.IO) { - ObjBox.boxStore.get().allEntityClasses.forEach { clazz -> - try { - val query = query(clazz) ?: throw Exception("No corresponding query") - val count = query.count() + private suspend inline fun upload(isUploadedProperty: Property, + stub: DataOperationsCoroutineGrpc.DataOperationsCoroutineStub) = coroutineScope { + try { + val query = query(isUploadedProperty, false) ?: throw Exception("No corresponding query") + val count = query.count() + Log.d("ZXCV", "$count") + val uploadedKeys = (0 until count step N_UPLOADS).mapNotNull { offset -> + val entities = if (T::class.java == SurveyEntity::class.java) { + query.find(offset, N_UPLOADS).filter { entity -> (entity as? SurveyEntity)?.isAvailable() == false } + } else { + query.find(offset, N_UPLOADS) + } - (0 until count step N_UPLOADS).forEach { offset -> - val entities = if (clazz == SurveyEntity::class.java) { - query.find(offset, N_UPLOADS).filter { entity -> (entity as? SurveyEntity)?.isAvailable() == false } - } else { - query.find(offset, N_UPLOADS) - } - val deferred = entities.map { entity -> - async { - try { - toProto(entity)?.let { stub.createDatum(it) } - entity - } catch (e: Exception) { - null - } + entities.map { entity -> + async { + try { + toProto(entity)?.let { proto -> stub.createDatum(proto) } + entity.id + } catch (e: Exception) { + null } } - val uploaded = deferred.awaitAll().filterNotNull() - uploaded.forEach { entity -> (entity as? Base)?.isUploaded = true } - ObjBox.put(uploaded) - } - } catch (e: Exception) { - AppLog.ee(e) - e.printStackTrace() - } - } - } - - private suspend fun removeAll() = withContext(Dispatchers.IO) { - ObjBox.boxStore.get().allEntityClasses.forEach { clazz -> - try { - remove(clazz) - } catch (e: Exception) { - AppLog.ee(e) - e.printStackTrace() - } - } - } - - private suspend fun terminate(channel: ManagedChannel) = withContext(Dispatchers.IO) { - try { - channel.shutdownNow().awaitTermination(10, TimeUnit.SECONDS) + }.awaitAll().filterNotNull() + }.flatten() + ObjBox.removeByKeys(uploadedKeys) } catch (e: Exception) { + AppLog.ee(e) e.printStackTrace() } } - @Suppress("UNCHECKED_CAST") - private fun query(clazz: Class): Query? { - val box = ObjBox.boxFor(clazz) ?: return null - val properties = (box.entityInfo as? EntityInfo)?.allProperties ?: return null - val isUploaded = properties.find { property -> property.name == "isUploaded" } - ?: return null - return box.query().equal(isUploaded, false).build() + + private suspend fun uploadAll(stub: DataOperationsCoroutineGrpc.DataOperationsCoroutineStub) { + upload(PhysicalActivityTransitionEntity_.isUploaded, stub) + upload(PhysicalActivityEntity_.isUploaded, stub) + upload(AppUsageEventEntity_.isUploaded, stub) + upload(BatteryEntity_.isUploaded, stub) + upload(BluetoothEntity_.isUploaded, stub) + upload(CallLogEntity_.isUploaded, stub) + upload(DeviceEventEntity_.isUploaded, stub) + upload(ExternalSensorEntity_.isUploaded, stub) + upload(InstalledAppEntity_.isUploaded, stub) + upload(KeyLogEntity_.isUploaded, stub) + upload(LocationEntity_.isUploaded, stub) + upload(MediaEntity_.isUploaded, stub) + upload(MessageEntity_.isUploaded, stub) + upload(NotificationEntity_.isUploaded, stub) + upload(PhysicalStatEntity_.isUploaded, stub) + upload(SensorEntity_.isUploaded, stub) + upload(SurveyEntity_.isUploaded, stub) + upload(DataTrafficEntity_.isUploaded, stub) + upload(WifiEntity_.isUploaded, stub) } - @Suppress("UNCHECKED_CAST") - private fun remove(clazz: Class) { - val box = ObjBox.boxFor(clazz) ?: return - val properties = (box.entityInfo as? EntityInfo)?.allProperties ?: return - val isUploaded = properties.find { property -> property.name == "isUploaded" } ?: return - box.query().equal(isUploaded, true).build().remove() + private inline fun query(isUploadedProperty: Property, isUploaded: Boolean) : Query? { + return ObjBox.boxFor()?.query()?.equal(isUploadedProperty, isUploaded)?.build() } private fun toProto(entity: T): DatumProto.Datum? { diff --git a/app/src/main/kotlin/kaist/iclab/abclogger/collector/Base.kt b/app/src/main/kotlin/kaist/iclab/abclogger/collector/Base.kt index cdbac9c..7268879 100644 --- a/app/src/main/kotlin/kaist/iclab/abclogger/collector/Base.kt +++ b/app/src/main/kotlin/kaist/iclab/abclogger/collector/Base.kt @@ -11,5 +11,5 @@ abstract class Base( var utcOffset: Float = Float.MIN_VALUE, var subjectEmail: String = "", var deviceInfo: String = "", - @Index var isUploaded: Boolean = false + var isUploaded: Boolean = false ) \ No newline at end of file diff --git a/app/src/main/kotlin/kaist/iclab/abclogger/collector/externalsensor/polar/PolarH10Collector.kt b/app/src/main/kotlin/kaist/iclab/abclogger/collector/externalsensor/polar/PolarH10Collector.kt index 99a8235..0375d21 100644 --- a/app/src/main/kotlin/kaist/iclab/abclogger/collector/externalsensor/polar/PolarH10Collector.kt +++ b/app/src/main/kotlin/kaist/iclab/abclogger/collector/externalsensor/polar/PolarH10Collector.kt @@ -61,7 +61,7 @@ class PolarH10Collector(val context: Context) : BaseCollector, PolarBleApiCallba name = "PolarH10", description = "ECG/mV", firstValue = ecg.toFloat() - ).fill(timeMillis = data.timeStamp) + ).fill(timeMillis = System.currentTimeMillis()) } }.buffer( 5, TimeUnit.SECONDS diff --git a/app/src/main/kotlin/kaist/iclab/abclogger/collector/keylog/KeyLogCollector.kt b/app/src/main/kotlin/kaist/iclab/abclogger/collector/keylog/KeyLogCollector.kt index d8b122f..5f2f1d7 100644 --- a/app/src/main/kotlin/kaist/iclab/abclogger/collector/keylog/KeyLogCollector.kt +++ b/app/src/main/kotlin/kaist/iclab/abclogger/collector/keylog/KeyLogCollector.kt @@ -92,7 +92,7 @@ class KeyLogCollector(val context: Context) : BaseCollector { currentKeyType = newKeyLog.type.name ).fill(timeMillis = eventTime).also { entity -> ObjBox.put(entity) - collector.setStatus(Status(lastTime = eventTime)) + collector.setStatus(Status(lastTime = System.currentTimeMillis())) } }