diff --git a/buildSrc/src/main/kotlin/org/jetbrains/packagesearch/gradle/GeneratePackageSearchObject.kt b/buildSrc/src/main/kotlin/org/jetbrains/packagesearch/gradle/GeneratePackageSearchObject.kt index 3bcbe9af..0734203e 100644 --- a/buildSrc/src/main/kotlin/org/jetbrains/packagesearch/gradle/GeneratePackageSearchObject.kt +++ b/buildSrc/src/main/kotlin/org/jetbrains/packagesearch/gradle/GeneratePackageSearchObject.kt @@ -41,6 +41,9 @@ open class GeneratePackageSearchObject @Inject constructor(objects: ObjectFactor @get:Input val packageName = objects.property() + @get:Input + val databaseVersion = objects.property() + @get:Input val objectName = objects.property() .convention("PackageSearch") @@ -90,6 +93,15 @@ open class GeneratePackageSearchObject @Inject constructor(objects: ObjectFactor ) .build() ) + .addProperty( + PropertySpec.builder("databaseVersion", Int::class) + .getter( + FunSpec.getterBuilder() + .addStatement("return %L", databaseVersion.get()) + .build() + ) + .build() + ) .build() ) .build() diff --git a/plugin/core/build.gradle.kts b/plugin/core/build.gradle.kts index c3c5786f..1387a551 100644 --- a/plugin/core/build.gradle.kts +++ b/plugin/core/build.gradle.kts @@ -52,6 +52,7 @@ tasks { pluginId = pkgsPluginId outputDir = generatedDir packageName = "com.jetbrains.packagesearch.plugin.core" + databaseVersion = 1 } sourcesJar { dependsOn(generatePluginDataSources) diff --git a/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/services/PackageSearchProjectCachesService.kt b/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/services/PackageSearchProjectCachesService.kt index 03d704e6..d8d3ae21 100644 --- a/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/services/PackageSearchProjectCachesService.kt +++ b/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/services/PackageSearchProjectCachesService.kt @@ -15,7 +15,7 @@ import kotlin.io.path.div class PackageSearchProjectCachesService(private val project: Project) : Disposable { private val cacheFilePath - get() = cachesDirectory / "db-${PackageSearch.pluginVersion}.db" + get() = cachesDirectory / "db-v${PackageSearch.databaseVersion}.db" private val cachesDirectory get() = project.getProjectDataPath("caches") / "packagesearch" diff --git a/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt b/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt index 506daa4a..320dc4a7 100644 --- a/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt +++ b/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt @@ -104,12 +104,11 @@ suspend fun Module.getDeclaredDependencies(): List + .map { declaredDependency -> PackageSearchGradleDeclaredPackage( id = declaredDependency.packageId, declaredVersion = declaredDependency.version?.let { NormalizedVersion.fromStringOrNull(it) }, - remoteInfo = remoteInfo[declaredDependency.packageId] as? ApiMavenPackage - ?: return@mapNotNull null, + remoteInfo = remoteInfo[declaredDependency.packageId] as? ApiMavenPackage, icon = remoteInfo[declaredDependency.packageId]?.icon ?: IconProvider.Icons.MAVEN, module = declaredDependency.groupId, diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt index ce920ea1..9c97cfac 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchApplicationCachesService.kt @@ -51,10 +51,7 @@ class PackageSearchApplicationCachesService : RecoveryAction, Disposable { companion object { private val cacheFilePath - get() = cacheDirectory / "db-${PackageSearch.pluginVersion}.db" - - private val cacheDirectory - get() = appSystemDir / "caches" / "packagesearch" + get() = appSystemDir / "caches" / "packagesearch" / "db-v${PackageSearch.databaseVersion}.db" } @PKGSInternalAPI diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt index 3c4d99e0..7d79be97 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt @@ -128,7 +128,6 @@ class PackageSearchProjectService(override val project: Project) : PackageSearch .flatMapLatest { moduleProvidersList } .retry(5) .onEach { logDebug("${this::class.qualifiedName}#modulesStateFlow") { "modules.size = ${it.size}" } } - .modifiedBy(restartFlow) { _, _ -> emptyList() } .stateIn(coroutineScope, SharingStarted.Eagerly, emptyList()) val modulesByBuildFile = modulesStateFlow diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt index 4229e1cd..9db8e34b 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/ApiPackageCacheEntry.kt @@ -10,9 +10,9 @@ import org.jetbrains.packagesearch.api.v3.http.SearchPackagesRequest @Serializable data class ApiPackageCacheEntry( - val data: ApiPackage?, - val packageId: String, - val packageIdHash: String, + val data: ApiPackage? = null, + val packageId: String? = null, + val packageIdHash: String? = null, @SerialName("_id") val id: Long? = null, val lastUpdate: Instant = Clock.System.now(), ) diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt index 73f10db9..6ded0313 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt @@ -3,8 +3,8 @@ package com.jetbrains.packagesearch.plugin.utils import com.jetbrains.packagesearch.plugin.core.nitrite.NitriteFilters import com.jetbrains.packagesearch.plugin.core.nitrite.coroutines.CoroutineObjectRepository import com.jetbrains.packagesearch.plugin.core.nitrite.insert +import com.jetbrains.packagesearch.plugin.core.utils.suspendSafe import korlibs.crypto.SHA256 -import kotlin.coroutines.cancellation.CancellationException import kotlin.random.Random import kotlin.time.Duration import kotlin.time.Duration.Companion.days @@ -16,7 +16,6 @@ import kotlinx.coroutines.sync.withLock import kotlinx.datetime.Clock import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json -import org.dizitart.no2.objects.ObjectFilter import org.jetbrains.packagesearch.api.v3.ApiPackage import org.jetbrains.packagesearch.api.v3.ApiRepository import org.jetbrains.packagesearch.api.v3.http.PackageSearchApi @@ -36,36 +35,33 @@ class PackageSearchApiPackageCache( override suspend fun getPackageInfoByIds(ids: Set) = getPackages( ids = ids, - apiCall = { apiClient.getPackageInfoByIds(it) }, - query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageId, it) }, useHashes = false ) override suspend fun getPackageInfoByIdHashes(ids: Set): Map = getPackages( ids = ids, - apiCall = { apiClient.getPackageInfoByIdHashes(it) }, - query = { NitriteFilters.Object.`in`(ApiPackageCacheEntry::packageIdHash, it) }, useHashes = true ) override suspend fun searchPackages(request: SearchPackagesRequest): List { val sha = SHA256.digest(Json.encodeToString(request).toByteArray()).base64 - logDebug("${this::class}#searchPackages") { "Searching for packages | searchSha = $sha" } + val contextName = "${Random.nextInt()} | ${this::class}#searchPackages" + logDebug(contextName) { "Searching for packages | searchSha = $sha" } val cachedEntry = searchCache.find(NitriteFilters.Object.eq(ApiSearchEntry::searchHash, sha)) .singleOrNull() if (cachedEntry != null) { val isOffline = !isOnline() val isCacheValid = cachedEntry.lastUpdate + maxAge > Clock.System.now() if (isOffline || isCacheValid) { - logDebug("${this::class}#searchPackages") { + logDebug(contextName) { "Using cached search results because `isOffline = $isOffline || isCacheValid = $isCacheValid` | searchSha = $sha" } return cachedEntry.packages } searchCache.remove(NitriteFilters.Object.eq(ApiSearchEntry::searchHash, sha)) } - logDebug("${this::class}#searchPackages") { "Fetching search results from the server | searchSha = $sha" } + logDebug(contextName) { "Fetching search results from the server | searchSha = $sha" } return apiClient.searchPackages(request) .also { searchCache.insert(ApiSearchEntry(it, sha, request)) } } @@ -85,41 +81,93 @@ class PackageSearchApiPackageCache( private suspend fun getPackages( ids: Set, - apiCall: suspend (Set) -> Map, - query: (Set) -> ObjectFilter, useHashes: Boolean, ): Map = cachesMutex.withLock { if (ids.isEmpty()) return emptyMap() - val localDatabaseResults = apiPackageCache.find(query(ids)) - .filter { if (isOnline()) Clock.System.now() < it.lastUpdate + maxAge else true } + val contextName = "${Random.nextInt()} | ${this::class.qualifiedName}#getPackages" + logDebug(contextName) { "Downloading packages | ids.size = ${ids.size}" } + val isOnlineStatus = isOnline() + val packageIdSelector = when { + useHashes -> ApiPackageCacheEntry::packageIdHash + else -> ApiPackageCacheEntry::packageId + } + val apiCall = when { + useHashes -> apiClient::getPackageInfoByIdHashes + else -> apiClient::getPackageInfoByIds + } + + // retrieve the packages from the local database + val localDatabaseResults = apiPackageCache + .find(NitriteFilters.Object.`in`(packageIdSelector, ids)) + .filter { if (isOnlineStatus) Clock.System.now() < it.lastUpdate + maxAge else true } .toList() + + // evaluate packages that are missing from the local database val missingIds = ids - when { - useHashes -> localDatabaseResults.map { it.packageIdHash }.toSet() - else -> localDatabaseResults.map { it.packageId }.toSet() + useHashes -> localDatabaseResults.mapNotNull { it.packageIdHash }.toSet() + else -> localDatabaseResults.mapNotNull { it.packageId }.toSet() } + logDebug(contextName) { "Missing packages | missingIds.size = ${missingIds.size}" } + // filter away packages that are unknown in our backend val localDatabaseResultsData = localDatabaseResults .mapNotNull { it.data } .associateBy { it.id } when { - missingIds.isEmpty() || !isOnline() -> localDatabaseResultsData + missingIds.isEmpty() || !isOnlineStatus -> { + logDebug(contextName) { + "Using cached packages only | isOnline = $isOnlineStatus, localDatabaseResultsData.size = ${localDatabaseResultsData.size}" + } + localDatabaseResultsData + } + else -> { + // retrieve the packages from the network val networkResults = runCatching { apiCall(missingIds) } - .onFailure { if (it is CancellationException) throw it } + .suspendSafe() .onFailure { logDebug("${this::class.qualifiedName}#getPackages", it) } - .getOrNull() - ?: emptyMap() - if (networkResults.isNotEmpty()) { - val packageEntries = networkResults.values.map { it.asCacheEntry() } - apiPackageCache.remove( - filter = NitriteFilters.Object.`in`( - path = ApiPackageCacheEntry::packageId, - value = packageEntries.map { it.packageId } + if (networkResults.isSuccess) { + val packageEntries = networkResults.getOrThrow() + .values + .map { it.asCacheEntry() } + if (packageEntries.isNotEmpty()) { + logDebug(contextName) { "No packages found | missingIds.size = ${missingIds.size}" } + + // remove the old entries + apiPackageCache.remove( + filter = NitriteFilters.Object.`in`( + path = packageIdSelector, + value = packageEntries.mapNotNull { it.packageId } + ) ) - ) - apiPackageCache.insert(packageEntries) + logDebug(contextName) { + "Removing old entries | packageEntries.size = ${packageEntries.size}" + } + } + // evaluate packages that are missing from our backend + val retrievedPackageIds = + packageEntries.mapNotNull { if (useHashes) it.packageIdHash else it.packageId } + .toSet() + val unknownPackages = missingIds.minus(retrievedPackageIds) + .map { id -> + when { + useHashes -> ApiPackageCacheEntry(packageIdHash = id) + else -> ApiPackageCacheEntry(packageId = id) + } + } + logDebug(contextName) { + "New unknown packages | unknownPackages.size = ${unknownPackages.size}" + } + // insert the new entries + val toInsert = packageEntries + unknownPackages + if (toInsert.isNotEmpty()) apiPackageCache.insert(toInsert) + } + val networkResultsData = networkResults.getOrDefault(emptyMap()) + logDebug(contextName) { + "Using network results and caches | networkResults.size = ${networkResultsData.size}, " + + "localDatabaseResultsData = ${localDatabaseResultsData.size}" } - localDatabaseResultsData + networkResults + localDatabaseResultsData + networkResultsData } } }