From 7aca6dbf1ac85e7f7204bda775968564eff0638d Mon Sep 17 00:00:00 2001 From: Lamberto Basti Date: Wed, 28 Feb 2024 17:36:42 +0100 Subject: [PATCH] Fix IDEA-347713 Package Search Plugin high CPU use (#111) This update modifies the caching system and retrieval of modules in Gradle projects. It revamps the caching of declared dependencies by associating them with their SHA hash of the respective build file, and storing them in the local caches. Co-authored-by: Lamberto Basti --- .../coroutines/CoroutineObjectRepository.kt | 3 +- plugin/core/build.gradle.kts | 2 +- .../plugin/core/extensions/Contexts.kt | 5 +- .../plugin/gradle/GradleModuleProvider.kt | 4 +- .../packagesearch/plugin/gradle/KMPUtils.kt | 190 +++++++++++++++++- .../KotlinMultiplatformModuleProvider.kt | 97 +-------- .../plugin/gradle/GradleDependencyModel.kt | 33 +-- .../plugin/gradle/utils/GradleUtils.kt | 84 +++++++- .../services/PackageSearchProjectService.kt | 26 +-- .../packagesearch/plugin/utils/Utils.kt | 15 ++ .../utils/WindowedModuleBuilderContext.kt | 2 - 11 files changed, 302 insertions(+), 159 deletions(-) diff --git a/nitrite/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/nitrite/coroutines/CoroutineObjectRepository.kt b/nitrite/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/nitrite/coroutines/CoroutineObjectRepository.kt index 9910cbc7..58921093 100644 --- a/nitrite/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/nitrite/coroutines/CoroutineObjectRepository.kt +++ b/nitrite/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/nitrite/coroutines/CoroutineObjectRepository.kt @@ -3,6 +3,7 @@ package com.jetbrains.packagesearch.plugin.core.nitrite.coroutines import com.jetbrains.packagesearch.plugin.core.nitrite.DocumentPathBuilder import com.jetbrains.packagesearch.plugin.core.nitrite.asKotlin import com.jetbrains.packagesearch.plugin.core.nitrite.serialization.NitriteDocumentFormat +import java.io.Closeable import kotlin.reflect.KProperty import kotlin.reflect.KType import kotlinx.coroutines.CoroutineDispatcher @@ -35,7 +36,7 @@ class CoroutineObjectRepository @InternalAPI constructor( val type: KType, private val documentFormat: NitriteDocumentFormat, override val dispatcher: CoroutineDispatcher = Dispatchers.IO, -) : CoroutineWrapper() { +) : CoroutineWrapper(), Closeable by synchronous { data class Change(val changeType: ChangeType, val changedItems: Flow) data class Item(val changeTimestamp: Instant, val changeType: ChangeType, val item: T) diff --git a/plugin/core/build.gradle.kts b/plugin/core/build.gradle.kts index 1387a551..f16feb9d 100644 --- a/plugin/core/build.gradle.kts +++ b/plugin/core/build.gradle.kts @@ -52,7 +52,7 @@ tasks { pluginId = pkgsPluginId outputDir = generatedDir packageName = "com.jetbrains.packagesearch.plugin.core" - databaseVersion = 1 + databaseVersion = 2 } sourcesJar { dependsOn(generatePluginDataSources) diff --git a/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/extensions/Contexts.kt b/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/extensions/Contexts.kt index 09d03977..ab5dea5e 100644 --- a/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/extensions/Contexts.kt +++ b/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/extensions/Contexts.kt @@ -23,10 +23,7 @@ interface PackageSearchApiPackagesContext { } interface PackageSearchModuleBuilderContext : - ProjectContext, PackageSearchKnownRepositoriesContext, PackageSearchApiPackagesContext { - val projectCaches: CoroutineNitrite - val applicationCaches: CoroutineNitrite - } + ProjectContext, PackageSearchKnownRepositoriesContext, PackageSearchApiPackagesContext interface ProjectContext { val project: Project diff --git a/plugin/gradle/base/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/GradleModuleProvider.kt b/plugin/gradle/base/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/GradleModuleProvider.kt index 1a6804b4..ae8b747e 100644 --- a/plugin/gradle/base/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/GradleModuleProvider.kt +++ b/plugin/gradle/base/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/GradleModuleProvider.kt @@ -33,7 +33,9 @@ class GradleModuleProvider : AbstractGradleModuleProvider() { val configurationNames = model.configurations .filter { it.canBeDeclared } .map { it.name } - val declaredDependencies = module.getDeclaredDependencies() + val declaredDependencies = model.buildFilePath + ?.let { module.getDeclaredDependencies(it) } + ?: emptyList() val packageSearchGradleModule = PackageSearchGradleModule( name = model.projectName, identity = PackageSearchModule.Identity( diff --git a/plugin/gradle/kmp/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/KMPUtils.kt b/plugin/gradle/kmp/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/KMPUtils.kt index 5e4c2bd7..24f9abec 100644 --- a/plugin/gradle/kmp/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/KMPUtils.kt +++ b/plugin/gradle/kmp/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/KMPUtils.kt @@ -1,12 +1,44 @@ package com.jetbrains.packagesearch.plugin.gradle +import com.intellij.openapi.Disposable +import com.intellij.openapi.application.readAction +import com.intellij.openapi.components.Service +import com.intellij.openapi.components.Service.Level +import com.intellij.openapi.components.service +import com.intellij.openapi.module.Module +import com.intellij.openapi.project.Project import com.intellij.packageSearch.mppDependencyUpdater.MppDependency +import com.intellij.packageSearch.mppDependencyUpdater.MppDependencyModifier import com.intellij.packageSearch.mppDependencyUpdater.resolved.MppCompilationInfoModel import com.jetbrains.packagesearch.plugin.core.data.EditModuleContext +import com.jetbrains.packagesearch.plugin.core.data.IconProvider import com.jetbrains.packagesearch.plugin.core.data.PackageSearchDeclaredPackage import com.jetbrains.packagesearch.plugin.core.data.PackageSearchModuleVariant +import com.jetbrains.packagesearch.plugin.core.extensions.PackageSearchModuleBuilderContext +import com.jetbrains.packagesearch.plugin.core.nitrite.NitriteFilters +import com.jetbrains.packagesearch.plugin.core.nitrite.insert +import com.jetbrains.packagesearch.plugin.core.utils.NioPathSerializer +import com.jetbrains.packagesearch.plugin.core.utils.PackageSearchProjectCachesService +import com.jetbrains.packagesearch.plugin.core.utils.icon import com.jetbrains.packagesearch.plugin.core.utils.parseAttributesFromRawStrings +import com.jetbrains.packagesearch.plugin.gradle.utils.GradleDependencyModelCacheEntry +import com.jetbrains.packagesearch.plugin.gradle.utils.getDeclaredDependencies +import com.jetbrains.packagesearch.plugin.gradle.utils.toGradleDependencyModel +import java.nio.file.Path +import korlibs.crypto.SHA256 import kotlin.contracts.contract +import kotlin.io.path.absolutePathString +import kotlin.io.path.isRegularFile +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.singleOrNull +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import org.jetbrains.packagesearch.api.v3.ApiMavenPackage +import org.jetbrains.packagesearch.api.v3.ApiPackage +import org.jetbrains.packagesearch.api.v3.search.buildPackageTypes +import org.jetbrains.packagesearch.api.v3.search.kotlinMultiplatform +import org.jetbrains.packagesearch.packageversionutils.normalization.NormalizedVersion fun Set.buildAttributes(): List { val rawStrings = this@buildAttributes.mapNotNull { @@ -25,7 +57,6 @@ fun Set.buildAttributes(): List { return when (this) { is PackageSearchModuleVariant.Attribute.StringAttribute -> setOf(value) @@ -107,4 +138,159 @@ fun PackageSearchKotlinMultiplatformDeclaredDependency.Maven.toMPPDependency() = artifactId = artifactId, version = declaredVersion?.versionName, configuration = configuration, - ) \ No newline at end of file + ) + + +context(PackageSearchModuleBuilderContext) +suspend fun Module.getKMPVariants( + compilationModel: Map>, + buildFilePath: Path?, + availableScopes: List, +): List = coroutineScope { + if (buildFilePath == null) return@coroutineScope emptyList() + + val dependenciesBlockVariant = async { + val declaredDependencies = getDeclaredDependencies(buildFilePath) + PackageSearchKotlinMultiplatformVariant.DependenciesBlock( + declaredDependencies = declaredDependencies.asKmpVariantDependencies(), + compatiblePackageTypes = buildPackageTypes { + mavenPackages() + gradlePackages { + isRootPublication = true + } + }, + availableScopes = availableScopes, + defaultScope = "implementation".takeIf { it in availableScopes } + ?: declaredDependencies.map { it.configuration } + .groupBy { it } + .mapValues { it.value.count() } + .entries + .maxByOrNull { it.value } + ?.key + ?: availableScopes.first() + ) + } + + val rawDeclaredSourceSetDependencies = getDependenciesBySourceSet(buildFilePath) + + val packageIds = rawDeclaredSourceSetDependencies + .values + .asSequence() + .flatten() + .distinct() + .map { it.packageId } + + val dependencyInfo = getPackageInfoByIdHashes(packageIds.map { ApiPackage.hashPackageId(it) }.toSet()) + + val declaredSourceSetDependencies = + rawDeclaredSourceSetDependencies + .mapValues { (sourceSetName, dependencies) -> + dependencies.map { artifactModel -> + PackageSearchKotlinMultiplatformDeclaredDependency.Maven( + id = artifactModel.packageId, + declaredVersion = artifactModel.version?.let { NormalizedVersion.fromStringOrNull(it) }, + remoteInfo = dependencyInfo[artifactModel.packageId] as? ApiMavenPackage, + declarationIndexes = artifactModel.indexes, + groupId = artifactModel.groupId, + artifactId = artifactModel.artifactId, + variantName = sourceSetName, + configuration = artifactModel.configuration, + icon = dependencyInfo[artifactModel.packageId]?.icon + ?: IconProvider.Icons.GRADLE + ) + } + } + val sourceSetVariants = compilationModel + .mapKeys { it.key } + .map { (sourceSetName, compilationTargets) -> + PackageSearchKotlinMultiplatformVariant.SourceSet( + name = sourceSetName, + declaredDependencies = declaredSourceSetDependencies[sourceSetName] ?: emptyList(), + attributes = compilationTargets.buildAttributes(), + compatiblePackageTypes = buildPackageTypes { + gradlePackages { + kotlinMultiplatform { + compilationTargets.forEach { compilationTarget -> + when { + compilationTarget is MppCompilationInfoModel.Js -> when (compilationTarget.compiler) { + MppCompilationInfoModel.Js.Compiler.IR -> jsIr() + MppCompilationInfoModel.Js.Compiler.LEGACY -> jsLegacy() + } + + compilationTarget is MppCompilationInfoModel.Native -> native(compilationTarget.target) + compilationTarget == MppCompilationInfoModel.Wasm -> wasm() + } + } + when { + MppCompilationInfoModel.Android in compilationTargets -> android() + MppCompilationInfoModel.Jvm in compilationTargets -> jvm() + } + } + } + }, + compilerTargets = compilationTargets + ) + } + + sourceSetVariants + dependenciesBlockVariant.await() +} + +@Serializable +data class GradleKMPDependencyModelCacheEntry( + @SerialName("_id") val id: Long? = null, + val buildFile: String, + val buildFileSha: String, + val dependencies: Map>, +) + +@Service(Level.PROJECT) +class GradleKMPCacheService(project: Project) : Disposable { + val kmpDependencyRepository = + project.PackageSearchProjectCachesService.getRepository("gradle-kmp-dependencies") + + override fun dispose() = kmpDependencyRepository.close() +} + +context(PackageSearchModuleBuilderContext) +private suspend fun Module.getDependenciesBySourceSet(buildFilePath: Path): Map> { + if (!buildFilePath.isRegularFile()) return emptyMap() + + val buildFileHash = SHA256() + .update(buildFilePath.toFile().readBytes()) + .digest() + .hex + + val entry = project.service() + .kmpDependencyRepository + .find( + filter = NitriteFilters.Object.eq( + path = GradleDependencyModelCacheEntry::buildFile, + value = buildFilePath.absolutePathString() + ) + ).singleOrNull() + + if (entry?.buildFileSha == buildFileHash) return entry.dependencies + + val filteredDependenciesBySourceSet = + MppDependencyModifier.dependenciesBySourceSet(this) + ?.filterNotNullValues() + ?.mapValues { readAction { it.value.artifacts().map { it.toGradleDependencyModel() } }.distinct() } + ?: emptyMap() + + project.service() + .kmpDependencyRepository + .update( + filter = NitriteFilters.Object.eq( + path = GradleDependencyModelCacheEntry::buildFile, + value = buildFilePath.absolutePathString() + ), + update = GradleKMPDependencyModelCacheEntry( + buildFile = buildFilePath.absolutePathString(), + buildFileSha = buildFileHash, + dependencies = filteredDependenciesBySourceSet + ), + upsert = true + ) + + return filteredDependenciesBySourceSet +} \ No newline at end of file diff --git a/plugin/gradle/kmp/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/KotlinMultiplatformModuleProvider.kt b/plugin/gradle/kmp/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/KotlinMultiplatformModuleProvider.kt index 12d60f11..4fdef872 100644 --- a/plugin/gradle/kmp/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/KotlinMultiplatformModuleProvider.kt +++ b/plugin/gradle/kmp/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/KotlinMultiplatformModuleProvider.kt @@ -20,6 +20,7 @@ import com.jetbrains.packagesearch.plugin.core.utils.icon import com.jetbrains.packagesearch.plugin.core.utils.toDirectory import com.jetbrains.packagesearch.plugin.gradle.utils.getDeclaredDependencies import com.jetbrains.packagesearch.plugin.gradle.utils.toGradleDependencyModel +import java.nio.file.Path import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.flow.FlowCollector @@ -41,6 +42,7 @@ class KotlinMultiplatformModuleProvider : AbstractGradleModuleProvider() { .collect { compilationModel -> val variants = module.getKMPVariants( compilationModel = compilationModel, + buildFilePath = model.buildFilePath, availableScopes = model.configurations .filter { it.canBeDeclared } .map { it.name } @@ -68,100 +70,5 @@ class KotlinMultiplatformModuleProvider : AbstractGradleModuleProvider() { } - context(PackageSearchModuleBuilderContext) - suspend fun Module.getKMPVariants( - compilationModel: Map>, - availableScopes: List, - ): List = coroutineScope { - val dependenciesBlockVariant = async { - val declaredDependencies = getDeclaredDependencies() - PackageSearchKotlinMultiplatformVariant.DependenciesBlock( - declaredDependencies = declaredDependencies.asKmpVariantDependencies(), - compatiblePackageTypes = buildPackageTypes { - mavenPackages() - gradlePackages { - isRootPublication = true - } - }, - availableScopes = availableScopes, - defaultScope = "implementation".takeIf { it in availableScopes } - ?: declaredDependencies.map { it.configuration } - .groupBy { it } - .mapValues { it.value.count() } - .entries - .maxByOrNull { it.value } - ?.key - ?: availableScopes.first() - ) - } - - val rawDeclaredSourceSetDependencies = MppDependencyModifier - .dependenciesBySourceSet(this@getKMPVariants) - ?.filterNotNullValues() - ?.mapValues { readAction { it.value.artifacts().map { it.toGradleDependencyModel() } }.distinct() } - ?: emptyMap() - - val packageIds = rawDeclaredSourceSetDependencies - .values - .asSequence() - .flatten() - .distinct() - .map { it.packageId } - - val dependencyInfo = getPackageInfoByIdHashes(packageIds.map { ApiPackage.hashPackageId(it) }.toSet()) - - val declaredSourceSetDependencies = - rawDeclaredSourceSetDependencies - .mapValues { (sourceSetName, dependencies) -> - dependencies.map { artifactModel -> - PackageSearchKotlinMultiplatformDeclaredDependency.Maven( - id = artifactModel.packageId, - declaredVersion = artifactModel.version?.let { NormalizedVersion.fromStringOrNull(it) }, - remoteInfo = dependencyInfo[artifactModel.packageId] as? ApiMavenPackage, - declarationIndexes = artifactModel.indexes, - groupId = artifactModel.groupId, - artifactId = artifactModel.artifactId, - variantName = sourceSetName, - configuration = artifactModel.configuration, - icon = dependencyInfo[artifactModel.packageId]?.icon - ?: Icons.GRADLE - ) - } - } - val sourceSetVariants = compilationModel - .mapKeys { it.key } - .map { (sourceSetName, compilationTargets) -> - PackageSearchKotlinMultiplatformVariant.SourceSet( - name = sourceSetName, - declaredDependencies = declaredSourceSetDependencies[sourceSetName] ?: emptyList(), - attributes = compilationTargets.buildAttributes(), - compatiblePackageTypes = buildPackageTypes { - gradlePackages { - kotlinMultiplatform { - compilationTargets.forEach { compilationTarget -> - when { - compilationTarget is Js -> when (compilationTarget.compiler) { - Js.Compiler.IR -> jsIr() - Js.Compiler.LEGACY -> jsLegacy() - } - - compilationTarget is Native -> native(compilationTarget.target) - compilationTarget == MppCompilationInfoModel.Wasm -> wasm() - } - } - when { - Android in compilationTargets -> android() - Jvm in compilationTargets -> jvm() - } - } - } - }, - compilerTargets = compilationTargets - ) - } - - sourceSetVariants + dependenciesBlockVariant.await() - } - } diff --git a/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/GradleDependencyModel.kt b/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/GradleDependencyModel.kt index e9ab513e..cca86f2c 100644 --- a/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/GradleDependencyModel.kt +++ b/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/GradleDependencyModel.kt @@ -1,39 +1,16 @@ package com.jetbrains.packagesearch.plugin.gradle import com.jetbrains.packagesearch.plugin.core.extensions.DependencyDeclarationIndexes +import kotlinx.serialization.Serializable +@Serializable data class GradleDependencyModel( val groupId: String, val artifactId: String, val version: String?, val configuration: String, val indexes: DependencyDeclarationIndexes, -) { +) - val packageId - get() = "maven:$groupId:$artifactId" - - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (javaClass != other?.javaClass) return false - - other as GradleDependencyModel - - if (groupId != other.groupId) return false - if (artifactId != other.artifactId) return false - if (version != other.version) return false - if (configuration != other.configuration) return false - - return true - } - - override fun hashCode(): Int { - var result = groupId.hashCode() - result = 31 * result + artifactId.hashCode() - result = 31 * result + (version?.hashCode() ?: 0) - result = 31 * result + configuration.hashCode() - return result - } - - -} \ No newline at end of file +val GradleDependencyModel.packageId + get() = "maven:$groupId:$artifactId" \ No newline at end of file diff --git a/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt b/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt index 712fe67d..4cc981a7 100644 --- a/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt +++ b/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt @@ -4,21 +4,34 @@ package com.jetbrains.packagesearch.plugin.gradle.utils import com.android.tools.idea.gradle.dsl.api.ProjectBuildModel import com.intellij.externalSystem.DependencyModifierService +import com.intellij.openapi.Disposable import com.intellij.openapi.application.readAction +import com.intellij.openapi.components.Service +import com.intellij.openapi.components.service import com.intellij.openapi.module.Module import com.intellij.openapi.project.Project import com.jetbrains.packagesearch.plugin.core.data.IconProvider import com.jetbrains.packagesearch.plugin.core.extensions.PackageSearchModuleBuilderContext +import com.jetbrains.packagesearch.plugin.core.nitrite.NitriteFilters +import com.jetbrains.packagesearch.plugin.core.nitrite.insert import com.jetbrains.packagesearch.plugin.core.utils.IntelliJApplication -import com.jetbrains.packagesearch.plugin.core.utils.asMavenApiPackage +import com.jetbrains.packagesearch.plugin.core.utils.NioPathSerializer +import com.jetbrains.packagesearch.plugin.core.utils.PackageSearchProjectCachesService import com.jetbrains.packagesearch.plugin.core.utils.filesChangedEventFlow import com.jetbrains.packagesearch.plugin.core.utils.icon import com.jetbrains.packagesearch.plugin.core.utils.mapUnit import com.jetbrains.packagesearch.plugin.core.utils.registryFlow import com.jetbrains.packagesearch.plugin.core.utils.watchExternalFileChanges +import com.jetbrains.packagesearch.plugin.gradle.GradleDependencyModel import com.jetbrains.packagesearch.plugin.gradle.PackageSearchGradleDeclaredPackage import com.jetbrains.packagesearch.plugin.gradle.PackageSearchGradleModel +import com.jetbrains.packagesearch.plugin.gradle.packageId +import java.nio.file.Path import java.nio.file.Paths +import korlibs.crypto.SHA256 +import kotlin.io.path.absolutePathString +import kotlin.io.path.isRegularFile +import kotlin.io.path.readBytes import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.filter @@ -26,6 +39,9 @@ import kotlinx.coroutines.flow.flatMapConcat import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.merge +import kotlinx.coroutines.flow.singleOrNull +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable import org.jetbrains.packagesearch.api.v3.ApiMavenPackage import org.jetbrains.packagesearch.api.v3.ApiPackage import org.jetbrains.packagesearch.api.v3.ApiRepository @@ -81,15 +97,73 @@ suspend fun Module.getDeclaredKnownRepositories(): Map { return knownRepositories.filterKeys { it in declaredDependencies } } +@Serializable +data class GradleDependencyModelCacheEntry( + @SerialName("_id") val id: Long? = null, + val buildFile: String, + val buildFileSha: String, + val dependencies: List, +) + context(PackageSearchModuleBuilderContext) -suspend fun Module.getDeclaredDependencies(): List { - val declaredDependencies = readAction { - ProjectBuildModel.get(project).getModuleBuildModel(this) +suspend fun retrieveGradleDependencyModel(nativeModule: Module, buildFile: Path): List { + if (!buildFile.isRegularFile()) { + return emptyList() + } + + val buildFileSha = SHA256.create() + .update(buildFile.readBytes()) + .digest() + .hex + + val cache = project.service() + .dependencyRepository + .find( + filter = NitriteFilters.Object.eq( + path = GradleDependencyModelCacheEntry::buildFile, + value = buildFile.absolutePathString() + ) + ).singleOrNull() + + if (cache?.buildFileSha == buildFileSha) return cache.dependencies + + val dependencies = readAction { + ProjectBuildModel.get(nativeModule.project).getModuleBuildModel(nativeModule) ?.dependencies() ?.artifacts() ?.map { it.toGradleDependencyModel() } ?: emptyList() - }.distinct() + } + + project.service() + .dependencyRepository + .update( + filter = NitriteFilters.Object.eq( + path = GradleDependencyModelCacheEntry::buildFile, + value = buildFile.absolutePathString() + ), + update = GradleDependencyModelCacheEntry( + buildFile = buildFile.absolutePathString(), + buildFileSha = buildFileSha, + dependencies = dependencies + ), + upsert = true + ) + + return dependencies +} + +@Service(Service.Level.PROJECT) +class GradleCacheService(project: Project) : Disposable { + val dependencyRepository = + project.PackageSearchProjectCachesService.getRepository("gradle-dependencies") + + override fun dispose() = dependencyRepository.close() +} + +context(PackageSearchModuleBuilderContext) +suspend fun Module.getDeclaredDependencies(buildFile: Path): List { + val declaredDependencies = retrieveGradleDependencyModel(this, buildFile) val distinctIds = declaredDependencies .asSequence() diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt index fe62b039..322cfa2b 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt @@ -18,20 +18,21 @@ import com.jetbrains.packagesearch.plugin.core.utils.replayOn import com.jetbrains.packagesearch.plugin.core.utils.toolWindowOpenedFlow import com.jetbrains.packagesearch.plugin.fus.PackageSearchFUSEvent import com.jetbrains.packagesearch.plugin.utils.PackageSearchApplicationCachesService -import com.jetbrains.packagesearch.plugin.utils.PackageSearchFUSService import com.jetbrains.packagesearch.plugin.utils.WindowedModuleBuilderContext +import com.jetbrains.packagesearch.plugin.utils.drop import com.jetbrains.packagesearch.plugin.utils.filterNotNullKeys import com.jetbrains.packagesearch.plugin.utils.logDebug import com.jetbrains.packagesearch.plugin.utils.logFUSEvent import com.jetbrains.packagesearch.plugin.utils.logWarn import com.jetbrains.packagesearch.plugin.utils.nativeModulesFlow import com.jetbrains.packagesearch.plugin.utils.startWithNull +import com.jetbrains.packagesearch.plugin.utils.throttle import com.jetbrains.packagesearch.plugin.utils.timer import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.asFlow @@ -43,7 +44,6 @@ import kotlinx.coroutines.flow.emptyFlow import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.flatMapLatest import kotlinx.coroutines.flow.flatMapMerge -import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.onEach @@ -93,8 +93,6 @@ class PackageSearchProjectService( knownRepositoriesGetter = { knownRepositories }, packagesCache = IntelliJApplication.PackageSearchApplicationCachesService.apiPackageCache, coroutineScope = coroutineScope, - projectCaches = project.PackageSearchProjectCachesService.cache, - applicationCaches = IntelliJApplication.PackageSearchApplicationCachesService.cache, ) val packagesBeingDownloadedFlow = context.getLoadingFLow() @@ -120,10 +118,7 @@ class PackageSearchProjectService( .debounce(1.seconds) .distinctUntilChanged() - private val restartFlow = restartChannel.consumeAsFlow() - .shareIn(coroutineScope, SharingStarted.Eagerly, 0) - - val modulesStateFlow = restartFlow + val modulesStateFlow = restartChannel.consumeAsFlow() .onStart { emit(Unit) } .flatMapLatest { moduleProvidersList } .retry(5) @@ -164,7 +159,9 @@ class PackageSearchProjectService( else -> emptyFlow() } } + .distinctUntilChanged() .filter { it } + .throttle(30.minutes) .onEach { restart() } .retry { logWarn("${this::class.simpleName}#isOnlineFlow", throwable = it) @@ -194,14 +191,3 @@ class PackageSearchProjectService( } -private fun Flow.drop(count: Int, function: (T) -> Boolean) = flow { - var current = 0 - collect { - if (current < count && function(it)) { - current++ - } else { - emit(it) - } - } -} - diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/Utils.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/Utils.kt index 12c9eea8..4c756f0f 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/Utils.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/Utils.kt @@ -8,6 +8,7 @@ import com.intellij.openapi.module.Module import com.intellij.openapi.module.ModuleManager import com.intellij.openapi.project.ModuleListener import com.intellij.openapi.project.Project +import com.intellij.platform.util.coroutines.flow.throttle import com.intellij.util.Function import com.jetbrains.packagesearch.plugin.core.utils.FlowWithInitialValue import com.jetbrains.packagesearch.plugin.core.utils.flow @@ -117,4 +118,18 @@ internal fun timer(interval: Duration, generate: suspend () -> T) = flow { emit(generate()) delay(interval) } +} + +fun Flow.throttle(timeMs: Duration) = + throttle(timeMs.inWholeMilliseconds) + +fun Flow.drop(count: Int, function: (T) -> Boolean) = flow { + var current = 0 + collect { + if (current < count && function(it)) { + current++ + } else { + emit(it) + } + } } \ No newline at end of file diff --git a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/WindowedModuleBuilderContext.kt b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/WindowedModuleBuilderContext.kt index c6af61e9..c24cb625 100644 --- a/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/WindowedModuleBuilderContext.kt +++ b/plugin/src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/WindowedModuleBuilderContext.kt @@ -39,8 +39,6 @@ class WindowedModuleBuilderContext( private val knownRepositoriesGetter: () -> Map, private val packagesCache: PackageSearchApi, override val coroutineScope: CoroutineScope, - override val projectCaches: CoroutineNitrite, - override val applicationCaches: CoroutineNitrite, ) : PackageSearchModuleBuilderContext { override val knownRepositories: Map