Introduced caching for reading Gradle file dependencies. (#121)

This change will avoid unnecessary file readings when they have not been changed, reducing CPU usages on startup.
JetBrains · Feb 29, 2024 · 48416c2 · 48416c2
1 parent c5585b6
commit 48416c2
Show file tree

Hide file tree

Showing 11 changed files with 67 additions and 74 deletions.
diff --git a/...n/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/utils/NioPathSerializer.kt b/...n/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/utils/NioPathSerializer.kt
@@ -2,6 +2,7 @@ package com.jetbrains.packagesearch.plugin.core.utils
 
 import java.nio.file.Path
 import java.nio.file.Paths
+import kotlin.io.path.Path
 import kotlin.io.path.absolutePathString
 import kotlinx.serialization.KSerializer
 import kotlinx.serialization.builtins.serializer
@@ -12,7 +13,7 @@ object NioPathSerializer : KSerializer<Path> {
     override val descriptor = String.serializer().descriptor
 
     override fun deserialize(decoder: Decoder): Path =
-        Paths.get(String.serializer().deserialize(decoder))
+        Path(String.serializer().deserialize(decoder))
 
     override fun serialize(encoder: Encoder, value: Path) {
         String.serializer().serialize(encoder, value.absolutePathString())

diff --git a/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/utils/Utils.kt b/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/utils/Utils.kt
@@ -24,6 +24,7 @@ import com.intellij.openapi.util.registry.Registry
 import com.intellij.openapi.util.registry.RegistryManager
 import com.intellij.openapi.util.registry.RegistryValue
 import com.intellij.openapi.util.registry.RegistryValueListener
+import com.intellij.openapi.vfs.AsyncFileListener
 import com.intellij.openapi.vfs.LocalFileSystem
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.VirtualFileEvent
@@ -39,20 +40,25 @@ import com.jetbrains.packagesearch.plugin.core.data.IconProvider
 import com.jetbrains.packagesearch.plugin.core.data.PackageSearchDeclaredMavenPackage
 import com.jetbrains.packagesearch.plugin.core.data.PackageSearchDeclaredPackage
 import com.jetbrains.packagesearch.plugin.core.services.PackageSearchProjectCachesService
+import java.nio.file.Files
 import java.nio.file.Path
 import kotlin.contracts.contract
 import kotlin.coroutines.cancellation.CancellationException
+import kotlin.coroutines.resume
 import kotlinx.coroutines.channels.ProducerScope
 import kotlinx.coroutines.channels.awaitClose
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.FlowCollector
 import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.asFlow
 import kotlinx.coroutines.flow.callbackFlow
 import kotlinx.coroutines.flow.channelFlow
+import kotlinx.coroutines.flow.flattenConcat
 import kotlinx.coroutines.flow.launchIn
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.merge
 import kotlinx.coroutines.flow.onEach
+import kotlinx.coroutines.suspendCancellableCoroutine
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import org.jetbrains.packagesearch.api.v3.ApiMavenPackage
@@ -109,13 +115,15 @@ fun <T : Any, R> MessageBus.bufferFlow(
 val filesChangedEventFlow: Flow<List<VFileEvent>>
     get() = callbackFlow {
         val disposable = Disposer.newDisposable()
-        VirtualFileManager.getInstance().addAsyncFileListener(
-            {
-                trySend(it.toList())
-                null
-            },
-            disposable
-        )
+        val fileListener: (events: List<VFileEvent>) -> AsyncFileListener.ChangeApplier = {
+            val changeApplier = object : AsyncFileListener.ChangeApplier {
+                override fun afterVfsChange() {
+                    trySend(it.toList())
+                }
+            }
+            changeApplier
+        }
+        VirtualFileManager.getInstance().addAsyncFileListener(fileListener, disposable)
         awaitClose { Disposer.dispose(disposable) }
     }
 
@@ -362,3 +370,6 @@ val Module.isSourceSet
     get() = ExternalSystemApiUtil.getExternalModuleType(this) == "sourceSet"
 
 fun <T> Result<T>.suspendSafe() = onFailure { if (it is CancellationException) throw it }
+
+fun Path.isSameFileAsSafe(other: Path): Boolean = kotlin.runCatching { Files.isSameFile(this, other) }
+    .getOrDefault(false)
diff --git a/...le/base/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/GradleModuleProvider.kt b/...le/base/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/GradleModuleProvider.kt
@@ -44,7 +44,7 @@ class GradleModuleProvider : AbstractGradleModuleProvider() {
                     projectDir = model.projectDir.toDirectory(),
                 ),
                 buildFilePath = model.buildFilePath,
-                declaredKnownRepositories = module.getDeclaredKnownRepositories(),
+                declaredKnownRepositories = module.getDeclaredKnownRepositories(model.repositories),
                 declaredDependencies = declaredDependencies,
                 availableKnownRepositories = availableKnownRepositories,
                 packageSearchModel = model,

diff --git a/plugin/gradle/kmp/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/KMPUtils.kt b/plugin/gradle/kmp/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/KMPUtils.kt
@@ -7,6 +7,7 @@ import com.intellij.openapi.components.Service.Level
 import com.intellij.openapi.components.service
 import com.intellij.openapi.module.Module
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.refreshAndFindVirtualFile
 import com.intellij.packageSearch.mppDependencyUpdater.MppDependency
 import com.intellij.packageSearch.mppDependencyUpdater.MppDependencyModifier
 import com.intellij.packageSearch.mppDependencyUpdater.resolved.MppCompilationInfoModel
@@ -26,6 +27,7 @@ import com.jetbrains.packagesearch.plugin.gradle.utils.getDeclaredDependencies
 import com.jetbrains.packagesearch.plugin.gradle.utils.toGradleDependencyModel
 import java.nio.file.Path
 import korlibs.crypto.SHA256
+import korlibs.crypto.sha512
 import kotlin.contracts.contract
 import kotlin.io.path.absolutePathString
 import kotlin.io.path.isRegularFile
@@ -253,12 +255,9 @@ class GradleKMPCacheService(project: Project) : Disposable {
 
 context(PackageSearchModuleBuilderContext)
 private suspend fun Module.getDependenciesBySourceSet(buildFilePath: Path): Map<String, List<GradleDependencyModel>> {
-    if (!buildFilePath.isRegularFile()) return emptyMap()
+    val vf = buildFilePath.refreshAndFindVirtualFile() ?: return emptyMap()
 
-    val buildFileHash = SHA256()
-        .update(buildFilePath.toFile().readBytes())
-        .digest()
-        .hex
+    val buildFileHash = vf.contentsToByteArray().sha512().hex
 
     val entry = project.service<GradleKMPCacheService>()
         .kmpDependencyRepository

diff --git a/...ain/kotlin/com/jetbrains/packagesearch/plugin/gradle/KotlinMultiplatformModuleProvider.kt b/...ain/kotlin/com/jetbrains/packagesearch/plugin/gradle/KotlinMultiplatformModuleProvider.kt
@@ -2,33 +2,14 @@
 
 package com.jetbrains.packagesearch.plugin.gradle
 
-import com.intellij.externalSystem.DependencyModifierService
-import com.intellij.openapi.application.readAction
 import com.intellij.openapi.module.Module
-import com.intellij.packageSearch.mppDependencyUpdater.MppDependencyModifier
-import com.intellij.packageSearch.mppDependencyUpdater.resolved.MppCompilationInfoModel
-import com.intellij.packageSearch.mppDependencyUpdater.resolved.MppCompilationInfoModel.Android
-import com.intellij.packageSearch.mppDependencyUpdater.resolved.MppCompilationInfoModel.Js
-import com.intellij.packageSearch.mppDependencyUpdater.resolved.MppCompilationInfoModel.Jvm
-import com.intellij.packageSearch.mppDependencyUpdater.resolved.MppCompilationInfoModel.Native
 import com.intellij.packageSearch.mppDependencyUpdater.resolved.MppCompilationInfoProvider
 import com.jetbrains.packagesearch.plugin.core.PackageSearch
-import com.jetbrains.packagesearch.plugin.core.data.IconProvider.Icons
 import com.jetbrains.packagesearch.plugin.core.data.PackageSearchModule
 import com.jetbrains.packagesearch.plugin.core.extensions.PackageSearchModuleBuilderContext
-import com.jetbrains.packagesearch.plugin.core.utils.icon
 import com.jetbrains.packagesearch.plugin.core.utils.toDirectory
-import com.jetbrains.packagesearch.plugin.gradle.utils.getDeclaredDependencies
-import com.jetbrains.packagesearch.plugin.gradle.utils.toGradleDependencyModel
-import java.nio.file.Path
-import kotlinx.coroutines.async
-import kotlinx.coroutines.coroutineScope
+import com.jetbrains.packagesearch.plugin.gradle.utils.getDeclaredKnownRepositories
 import kotlinx.coroutines.flow.FlowCollector
-import org.jetbrains.packagesearch.api.v3.ApiMavenPackage
-import org.jetbrains.packagesearch.api.v3.ApiPackage
-import org.jetbrains.packagesearch.api.v3.search.buildPackageTypes
-import org.jetbrains.packagesearch.api.v3.search.kotlinMultiplatform
-import org.jetbrains.packagesearch.packageversionutils.normalization.NormalizedVersion
 
 class KotlinMultiplatformModuleProvider : AbstractGradleModuleProvider() {
 
@@ -55,11 +36,7 @@ class KotlinMultiplatformModuleProvider : AbstractGradleModuleProvider() {
                             projectDir = model.projectDir.toDirectory(),
                         ),
                         buildFilePath = model.buildFilePath,
-                        declaredKnownRepositories = knownRepositories - DependencyModifierService
-                            .getInstance(project)
-                            .declaredRepositories(module)
-                            .mapNotNull { it.id }
-                            .toSet(),
+                        declaredKnownRepositories = module.getDeclaredKnownRepositories(model.repositories),
                         variants = variants,
                         packageSearchModel = model,
                         availableKnownRepositories = knownRepositories,

diff --git a/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt b/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/GradleUtils.kt
@@ -14,30 +14,28 @@ import com.intellij.openapi.externalSystem.service.internal.ExternalSystemProces
 import com.intellij.openapi.externalSystem.service.internal.ExternalSystemResolveProjectTask
 import com.intellij.openapi.module.Module
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.refreshAndFindVirtualFile
+import com.intellij.openapi.vfs.toNioPathOrNull
 import com.jetbrains.packagesearch.plugin.core.data.IconProvider
 import com.jetbrains.packagesearch.plugin.core.extensions.PackageSearchModuleBuilderContext
 import com.jetbrains.packagesearch.plugin.core.nitrite.NitriteFilters
 import com.jetbrains.packagesearch.plugin.core.utils.IntelliJApplication
 import com.jetbrains.packagesearch.plugin.core.utils.PackageSearchProjectCachesService
 import com.jetbrains.packagesearch.plugin.core.utils.filesChangedEventFlow
 import com.jetbrains.packagesearch.plugin.core.utils.icon
+import com.jetbrains.packagesearch.plugin.core.utils.isSameFileAsSafe
 import com.jetbrains.packagesearch.plugin.core.utils.mapUnit
-import com.jetbrains.packagesearch.plugin.core.utils.registryFlow
 import com.jetbrains.packagesearch.plugin.core.utils.watchExternalFileChanges
 import com.jetbrains.packagesearch.plugin.gradle.GradleDependencyModel
 import com.jetbrains.packagesearch.plugin.gradle.PackageSearchGradleDeclaredPackage
 import com.jetbrains.packagesearch.plugin.gradle.PackageSearchGradleModel
 import com.jetbrains.packagesearch.plugin.gradle.packageId
 import java.nio.file.Path
 import java.nio.file.Paths
-import korlibs.crypto.SHA256
+import korlibs.crypto.sha512
 import kotlin.io.path.absolutePathString
-import kotlin.io.path.isRegularFile
-import kotlin.io.path.readBytes
 import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.asFlow
 import kotlinx.coroutines.flow.filter
-import kotlinx.coroutines.flow.flatMapConcat
 import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.merge
@@ -61,10 +59,8 @@ val globalGradlePropertiesPath
 val knownGradleAncillaryFilesFiles
     get() = listOf("gradle.properties", "local.properties", "gradle/libs.versions.toml")
 
-fun getModuleChangesFlow(
-    model: PackageSearchGradleModel,
-): Flow<Unit> {
-    val allFiles = buildSet {
+fun getModuleChangesFlow(model: PackageSearchGradleModel): Flow<Unit> {
+    val knownFiles = buildSet {
         if (model.buildFilePath != null) {
             add(model.buildFilePath)
         }
@@ -79,24 +75,22 @@ fun getModuleChangesFlow(
     }
 
     val buildFileChanges = filesChangedEventFlow
-        .flatMapConcat { it.map { it.path }.asFlow() }
-        .map { Paths.get(it) }
-        .filter { filePath -> allFiles.any { filePath == it } }
+        .map { it.mapNotNull { it.file?.toNioPathOrNull() } }
+        .filter { changes -> changes.any { change -> knownFiles.any { it.isSameFileAsSafe(change) } } }
         .mapUnit()
 
     return merge(
         watchExternalFileChanges(globalGradlePropertiesPath),
         buildFileChanges,
-        IntelliJApplication.registryFlow("packagesearch.sonatype.api.client").mapUnit(),
     )
 }
 
 context(PackageSearchModuleBuilderContext)
-suspend fun Module.getDeclaredKnownRepositories(): Map<String, ApiRepository> {
+suspend fun Module.getDeclaredKnownRepositories(repositories: List<String>): Map<String, ApiRepository> {
     val declaredDependencies = readAction {
         DependencyModifierService.getInstance(project).declaredRepositories(this)
     }.mapNotNull { it.id }
-    return knownRepositories.filterKeys { it in declaredDependencies }
+    return knownRepositories.filterKeys { it in declaredDependencies } + knownRepositories.filterValues { it.url in repositories }
 }
 
 @Serializable
@@ -109,14 +103,9 @@ data class GradleDependencyModelCacheEntry(
 
 context(PackageSearchModuleBuilderContext)
 suspend fun retrieveGradleDependencyModel(nativeModule: Module, buildFile: Path): List<GradleDependencyModel> {
-    if (!buildFile.isRegularFile()) {
-        return emptyList()
-    }
+    val vf = buildFile.refreshAndFindVirtualFile() ?: return emptyList()
 
-    val buildFileSha = SHA256.create()
-        .update(buildFile.readBytes())
-        .digest()
-        .hex
+    val buildFileSha = vf.contentsToByteArray().sha512().hex
 
     val cache = project.service<GradleCacheService>()
         .dependencyRepository

diff --git a/plugin/maven/src/main/kotlin/com/jetbrains/packagesearch/plugin/maven/MavenModuleProvider.kt b/plugin/maven/src/main/kotlin/com/jetbrains/packagesearch/plugin/maven/MavenModuleProvider.kt
@@ -4,12 +4,12 @@ package com.jetbrains.packagesearch.plugin.maven
 
 import com.intellij.openapi.components.service
 import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.toNioPathOrNull
 import com.jetbrains.packagesearch.plugin.core.data.PackageSearchModule
 import com.jetbrains.packagesearch.plugin.core.extensions.PackageSearchModuleBuilderContext
 import com.jetbrains.packagesearch.plugin.core.extensions.PackageSearchModuleProvider
 import com.jetbrains.packagesearch.plugin.core.utils.isSourceSet
 import com.jetbrains.packagesearch.plugin.core.utils.smartModeFlow
-import java.nio.file.Paths
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.asStateFlow
 import kotlinx.coroutines.flow.emptyFlow
@@ -26,8 +26,11 @@ class MavenModuleProvider : PackageSearchModuleProvider {
         else -> project.smartModeFlow.take(1).flatMapLatest {
             when (val mavenProject = project.findMavenProjectFor(nativeModule)) {
                 null -> emptyFlow()
-                else -> getModuleChangesFlow(Paths.get(mavenProject.file.path))
-                    .map { nativeModule.toPackageSearch(mavenProject) }
+                else -> when (val mavenProjectPath = mavenProject.file.toNioPathOrNull()) {
+                    null -> emptyFlow()
+                    else -> getModuleChangesFlow(mavenProjectPath)
+                        .map { nativeModule.toPackageSearch(mavenProject) }
+                }
             }
         }
     }

diff --git a/plugin/maven/src/main/kotlin/com/jetbrains/packagesearch/plugin/maven/MavenUtils.kt b/plugin/maven/src/main/kotlin/com/jetbrains/packagesearch/plugin/maven/MavenUtils.kt
@@ -7,6 +7,7 @@ import com.intellij.openapi.application.readAction
 import com.intellij.openapi.module.Module
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.toNioPathOrNull
 import com.intellij.psi.xml.XmlText
 import com.jetbrains.packagesearch.plugin.core.data.EditModuleContext
 import com.jetbrains.packagesearch.plugin.core.data.IconProvider
@@ -19,6 +20,7 @@ import com.jetbrains.packagesearch.plugin.core.utils.asMavenApiPackage
 import com.jetbrains.packagesearch.plugin.core.utils.filesChangedEventFlow
 import com.jetbrains.packagesearch.plugin.core.utils.flow
 import com.jetbrains.packagesearch.plugin.core.utils.icon
+import com.jetbrains.packagesearch.plugin.core.utils.isSameFileAsSafe
 import com.jetbrains.packagesearch.plugin.core.utils.mapUnit
 import com.jetbrains.packagesearch.plugin.core.utils.registryFlow
 import com.jetbrains.packagesearch.plugin.core.utils.toDirectory
@@ -27,6 +29,7 @@ import java.io.File
 import java.nio.file.Path
 import java.nio.file.Paths
 import kotlin.io.path.Path
+import kotlin.io.path.isSameFileAs
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.asFlow
 import kotlinx.coroutines.flow.filter
@@ -74,11 +77,9 @@ fun getModuleChangesFlow(pomPath: Path): Flow<Unit> = merge(
     watchExternalFileChanges(mavenSettingsFilePath),
     project.mavenImportFlow,
     filesChangedEventFlow
-        .flatMapLatest { it.map { it.path }.asFlow() }
-        .map { Paths.get(it) }
-        .filter { it == pomPath }
+        .map { it.mapNotNull { it.file?.toNioPathOrNull() } }
+        .filter { it.any { it.isSameFileAsSafe(pomPath) } }
         .mapUnit(),
-    IntelliJApplication.registryFlow("packagesearch.sonatype.api.client").mapUnit()
 )
 
 val xml = XML {

diff --git a/...in/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchFUSService.kt b/...in/src/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchFUSService.kt
@@ -7,17 +7,21 @@ import com.jetbrains.packagesearch.plugin.fus.log
 import com.jetbrains.packagesearch.plugin.utils.logWarn
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.flow.SharingStarted
 import kotlinx.coroutines.flow.consumeAsFlow
 import kotlinx.coroutines.flow.launchIn
 import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.flow.retry
+import kotlinx.coroutines.flow.shareIn
 
 @Service(Level.APP)
 class PackageSearchFUSService(coroutineScope: CoroutineScope) {
-    private val eventsChannel: Channel<PackageSearchFUSEvent> = Channel(capacity = Channel.UNLIMITED)
+    private val fusEventsChannel: Channel<PackageSearchFUSEvent> = Channel()
+    private val fusEventsFlow = fusEventsChannel.consumeAsFlow()
+        .shareIn(coroutineScope, SharingStarted.Lazily)
 
     init {
-        eventsChannel.consumeAsFlow()
+        fusEventsFlow
             .onEach { it.log() }
             .retry {
                 logWarn("${this::class.qualifiedName}#eventReportingJob", it) { "Failed to log FUS" }
@@ -27,6 +31,6 @@ class PackageSearchFUSService(coroutineScope: CoroutineScope) {
     }
 
     fun logEvent(event: PackageSearchFUSEvent) {
-        eventsChannel.trySend(event)
+        fusEventsChannel.trySend(event)
     }
 }
diff --git a/...rc/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt b/...rc/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt
@@ -117,10 +117,16 @@ class PackageSearchProjectService(
             .debounce(1.seconds)
             .distinctUntilChanged()
 
-    val modulesStateFlow = restartChannel.consumeAsFlow()
+    private val restartFlow = restartChannel.consumeAsFlow()
+        .shareIn(coroutineScope, SharingStarted.Lazily, 0)
+
+    val modulesStateFlow = restartFlow
         .onStart { emit(Unit) }
         .flatMapLatest { moduleProvidersList }
-        .retry(5)
+        .retry(5) {
+            logWarn("${this::class.simpleName}#modulesStateFlow", throwable = it)
+            true
+        }
         .onEach { logDebug("${this::class.qualifiedName}#modulesStateFlow") { "modules.size = ${it.size}" } }
         .stateIn(coroutineScope, SharingStarted.Lazily, emptyList())
 

diff --git a/...n/kotlin/com/jetbrains/packagesearch/plugin/ui/model/packageslist/PackageListViewModel.kt b/...n/kotlin/com/jetbrains/packagesearch/plugin/ui/model/packageslist/PackageListViewModel.kt
@@ -26,6 +26,7 @@ import com.jetbrains.packagesearch.plugin.utils.logFUSEvent
 import com.jetbrains.packagesearch.plugin.utils.logTODO
 import com.jetbrains.packagesearch.plugin.utils.logWarn
 import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.Duration.Companion.seconds
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.channels.Channel
@@ -94,6 +95,7 @@ class PackageListViewModel(
     init {
         searchQueryStateFlow
             .filter { it.isNotEmpty() }
+            .debounce(1.seconds)
             .onEach { logFUSEvent(PackageSearchFUSEvent.SearchRequest(it)) }
             .launchIn(viewModelScope)