233 - Improved services lazyness and reduced HTTP client retry attemp…

…ts. (#74) * Improved services lazyness and reduced HTTP client retry attempts. * Remove unused gradleIdentityPathOrNull utility This commit removes the unused 'gradleIdentityPathOrNull' utility from the Utils.kt file. This method was previously used to retrieve Gradle module identity path, but is no longer needed. The appearance and readability of the code have significantly improved as a result. * Refactor PackageSearchApiPackageCache handling The code for handling the network results in the PackageSearchApiPackageCache class has been refactored. Instead of performing a removal of old entries followed by an insertion of new entries, the code now performs an update operation for each new entry. This makes the handling of network results more efficient and concise.
JetBrains · Feb 19, 2024 · bb735c7 · bb735c7
1 parent 70ea0b2
commit bb735c7
Show file tree

Hide file tree

Showing 7 changed files with 141 additions and 87 deletions.
diff --git a/package-search-api-models b/package-search-api-models
diff --git a/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/utils/Utils.kt b/plugin/core/src/main/kotlin/com/jetbrains/packagesearch/plugin/core/utils/Utils.kt
@@ -27,6 +27,8 @@ import com.intellij.openapi.vfs.VirtualFileEvent
 import com.intellij.openapi.vfs.VirtualFileListener
 import com.intellij.openapi.vfs.VirtualFileManager
 import com.intellij.openapi.vfs.newvfs.events.VFileEvent
+import com.intellij.openapi.wm.ToolWindowManager
+import com.intellij.openapi.wm.ex.ToolWindowManagerListener
 import com.intellij.util.application
 import com.intellij.util.messages.MessageBus
 import com.intellij.util.messages.Topic
@@ -44,7 +46,6 @@ import kotlinx.coroutines.flow.FlowCollector
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.callbackFlow
 import kotlinx.coroutines.flow.channelFlow
-import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.flow.launchIn
 import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.merge
@@ -68,6 +69,40 @@ fun <T : Any, R> MessageBus.flow(
     awaitClose { connection.disconnect() }
 }
 
+sealed interface FlowEvent<T> {
+
+    @JvmInline
+    value class Added<T>(val item: T) : FlowEvent<T>
+
+    @JvmInline
+    value class Removed<T>(val item: T) : FlowEvent<T>
+
+    @JvmInline
+    value class Initial<T>(val items: List<T>) : FlowEvent<T>
+}
+
+fun <T : Any, R> MessageBus.bufferFlow(
+    topic: Topic<T>,
+    initialValue: (() -> List<R>)? = null,
+    listener: ProducerScope<FlowEvent<R>>.() -> T,
+) = channelFlow {
+    val buffer = mutableSetOf<R>()
+    flow(topic, listener).onEach { event ->
+        when (event) {
+            is FlowEvent.Added -> buffer.add(event.item)
+            is FlowEvent.Removed -> buffer.remove(event.item)
+            is FlowEvent.Initial -> {
+                buffer.clear()
+                buffer.addAll(event.items)
+            }
+        }
+        send(buffer.toList())
+    }
+        .launchIn(this)
+    initialValue?.invoke()?.let { send(it) }
+    awaitClose()
+}
+
 val filesChangedEventFlow: Flow<List<VFileEvent>>
     get() = callbackFlow {
         val disposable = Disposer.newDisposable()
@@ -173,43 +208,52 @@ fun <T> Flow<T>.replayOn(vararg replayFlows: Flow<*>) = channelFlow {
     merge(*replayFlows).collect { mutex.withLock { last?.let { send(it) } } }
 }
 
-val Project.fileOpenedFlow: Flow<List<VirtualFile>>
-    get() {
-        val flow = flow {
-            val buffer: MutableList<VirtualFile> = FileEditorManager.getInstance(this@fileOpenedFlow).openFiles
-                .toMutableList()
-            emit(buffer.toList())
-            messageBus.flow(FileEditorManagerListener.FILE_EDITOR_MANAGER) {
-                object : FileEditorManagerListener {
-                    override fun fileOpened(source: FileEditorManager, file: VirtualFile) {
-                        trySend(FileEditorEvent.FileOpened(file))
-                    }
-
-                    override fun fileClosed(source: FileEditorManager, file: VirtualFile) {
-                        trySend(FileEditorEvent.FileClosed(file))
-                    }
-                }
-            }.collect {
-                when (it) {
-                    is FileEditorEvent.FileClosed -> buffer.remove(it.file)
-                    is FileEditorEvent.FileOpened -> buffer.add(it.file)
-                }
-                emit(buffer.toList())
-            }
+fun Project.toolWindowOpenedFlow(toolWindowId: String): Flow<Boolean> = callbackFlow {
+    val manager = ToolWindowManager.getInstance(this@toolWindowOpenedFlow)
+    val toolWindow = manager.getToolWindow(toolWindowId)
+
+    // Initial state
+    trySend(toolWindow?.isVisible ?: false)
+
+    val listener = object : ToolWindowManagerListener {
+        override fun stateChanged(toolWindowManager: ToolWindowManager) {
+            trySend(manager.getToolWindow(toolWindowId)?.isVisible ?: false)
         }
-        return flow.withInitialValue(FileEditorManager.getInstance(this@fileOpenedFlow).openFiles.toList())
     }
 
-internal sealed interface FileEditorEvent {
+    // Register the listener
+    val connection = messageBus.connect()
+    connection.subscribe(ToolWindowManagerListener.TOPIC, listener)
 
-    val file: VirtualFile
+    // Cleanup on close
+    awaitClose { connection.disconnect() }
+}
 
-    @JvmInline
-    value class FileOpened(override val file: VirtualFile) : FileEditorEvent
+// Usage:
+// val toolWindowFlow = project.toolWindowOpenedFlow("YourToolWindowId")
+// toolWindowFlow.collect { isOpen ->
+//     println("Tool window is open: $isOpen")
+// }
+
+
+val Project.fileOpenedFlow
+    get() = messageBus.bufferFlow(
+        topic = FileEditorManagerListener.FILE_EDITOR_MANAGER,
+        initialValue = { FileEditorManager.getInstance(this).openFiles.toList() }
+    ) {
+        object : FileEditorManagerListener {
+            override fun fileOpened(source: FileEditorManager, file: VirtualFile) {
+                trySend(FlowEvent.Added(file))
+            }
 
-    @JvmInline
-    value class FileClosed(override val file: VirtualFile) : FileEditorEvent
-}
+            override fun fileClosed(source: FileEditorManager, file: VirtualFile) {
+                trySend(FlowEvent.Removed(file))
+            }
+        }
+    }
+
+val Project.project
+    get() = this
 
 val <T : Any> ExtensionPointName<T>.availableExtensionsFlow: FlowWithInitialValue<List<T>>
     get() {

diff --git a/...kotlin/com/jetbrains/packagesearch/plugin/gradle/PackageSearchProjectResolverExtension.kt b/...kotlin/com/jetbrains/packagesearch/plugin/gradle/PackageSearchProjectResolverExtension.kt
@@ -5,6 +5,7 @@ import com.intellij.openapi.externalSystem.model.project.ModuleData
 import com.jetbrains.packagesearch.plugin.gradle.PackageSearchGradleModel.Configuration
 import com.jetbrains.packagesearch.plugin.gradle.PackageSearchGradleModel.Dependency
 import com.jetbrains.packagesearch.plugin.gradle.tooling.PackageSearchGradleJavaModel
+import com.jetbrains.packagesearch.plugin.gradle.tooling.PackageSearchGradleModelBuilder
 import java.nio.file.Paths
 import org.gradle.tooling.model.idea.IdeaModule
 import org.jetbrains.plugins.gradle.service.project.AbstractProjectResolverExtension
@@ -15,7 +16,7 @@ class PackageSearchProjectResolverExtension : AbstractProjectResolverExtension()
         setOf(PackageSearchGradleJavaModel::class.java)
 
     override fun getToolingExtensionsClasses() =
-        setOf(com.jetbrains.packagesearch.plugin.gradle.tooling.PackageSearchGradleModelBuilder::class.java)
+        setOf(PackageSearchGradleModelBuilder::class.java)
 
     private inline fun <reified T> IdeaModule.getExtraProject(): T? =
         resolverCtx.getExtraProject(this@getExtraProject, T::class.java)

diff --git a/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/Utils.kt b/plugin/gradle/src/main/kotlin/com/jetbrains/packagesearch/plugin/gradle/utils/Utils.kt
@@ -20,12 +20,6 @@ val Module.isGradleSourceSet: Boolean
         return ExternalSystemApiUtil.getExternalModuleType(this) == GradleConstants.GRADLE_SOURCE_SET_MODULE_TYPE_KEY
     }
 
-val Module.gradleIdentityPathOrNull: String?
-    get() = CachedModuleDataFinder.getInstance(project)
-        .findMainModuleData(this)
-        ?.data
-        ?.gradleIdentityPathOrNull
-
 suspend fun Project.awaitExternalSystemInitialization() = suspendCoroutine {
     ExternalProjectsManager.getInstance(this@awaitExternalSystemInitialization)
         .runWhenInitialized { it.resume(Unit) }

diff --git a/...rc/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt b/...rc/main/kotlin/com/jetbrains/packagesearch/plugin/services/PackageSearchProjectService.kt
@@ -15,9 +15,8 @@ import com.jetbrains.packagesearch.plugin.core.utils.IntelliJApplication
 import com.jetbrains.packagesearch.plugin.core.utils.PackageSearchProjectCachesService
 import com.jetbrains.packagesearch.plugin.core.utils.fileOpenedFlow
 import com.jetbrains.packagesearch.plugin.core.utils.replayOn
-import com.jetbrains.packagesearch.plugin.core.utils.withInitialValue
+import com.jetbrains.packagesearch.plugin.core.utils.toolWindowOpenedFlow
 import com.jetbrains.packagesearch.plugin.fus.logOnlyStableToggle
-import com.jetbrains.packagesearch.plugin.ui.model.packageslist.modifiedBy
 import com.jetbrains.packagesearch.plugin.utils.PackageSearchApplicationCachesService
 import com.jetbrains.packagesearch.plugin.utils.WindowedModuleBuilderContext
 import com.jetbrains.packagesearch.plugin.utils.filterNotNullKeys
@@ -38,7 +37,7 @@ import kotlinx.coroutines.flow.combine
 import kotlinx.coroutines.flow.consumeAsFlow
 import kotlinx.coroutines.flow.debounce
 import kotlinx.coroutines.flow.distinctUntilChanged
-import kotlinx.coroutines.flow.drop
+import kotlinx.coroutines.flow.emptyFlow
 import kotlinx.coroutines.flow.filter
 import kotlinx.coroutines.flow.flatMapLatest
 import kotlinx.coroutines.flow.flatMapMerge
@@ -48,12 +47,8 @@ import kotlinx.coroutines.flow.map
 import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.flow.onStart
 import kotlinx.coroutines.flow.retry
-import kotlinx.coroutines.flow.retryWhen
 import kotlinx.coroutines.flow.shareIn
 import kotlinx.coroutines.flow.stateIn
-import kotlinx.coroutines.flow.transform
-import kotlinx.coroutines.sync.Mutex
-import kotlinx.coroutines.sync.withLock
 import org.jetbrains.packagesearch.api.v3.ApiRepository
 
 @Service(Level.PROJECT)
@@ -74,7 +69,7 @@ class PackageSearchProjectService(
     val isProjectExecutingSyncStateFlow = PackageSearchModuleBaseTransformerUtils.extensionsFlow
         .map { it.map { it.getSyncStateFlow(project) } }
         .flatMapLatest { combine(it) { it.all { it } } }
-        .stateIn(coroutineScope, SharingStarted.Eagerly, false)
+        .stateIn(coroutineScope, SharingStarted.Lazily, false)
 
     private val knownRepositoriesStateFlow = timer(12.hours) {
         IntelliJApplication.PackageSearchApplicationCachesService
@@ -103,7 +98,7 @@ class PackageSearchProjectService(
     val packagesBeingDownloadedFlow = context.getLoadingFLow()
         .distinctUntilChanged()
         .onEach { logDebug("${this::class.qualifiedName}#packagesBeingDownloadedFlow") { "$it" } }
-        .stateIn(coroutineScope, SharingStarted.Eagerly, false)
+        .stateIn(coroutineScope, SharingStarted.Lazily, false)
 
     private val moduleProvidersList
         get() = combine(
@@ -131,15 +126,23 @@ class PackageSearchProjectService(
         .flatMapLatest { moduleProvidersList }
         .retry(5)
         .onEach { logDebug("${this::class.qualifiedName}#modulesStateFlow") { "modules.size = ${it.size}" } }
-        .stateIn(coroutineScope, SharingStarted.Eagerly, emptyList())
+        .stateIn(coroutineScope, SharingStarted.Lazily, emptyList())
 
     val modulesByBuildFile = modulesStateFlow
         .map { it.associateBy { it.buildFilePath }.filterNotNullKeys() }
-        .stateIn(coroutineScope, SharingStarted.Eagerly, emptyMap())
+        .stateIn(coroutineScope, SharingStarted.Lazily, emptyMap())
 
     val modulesByIdentity = modulesStateFlow
         .map { it.associateBy { it.identity } }
-        .stateIn(coroutineScope, SharingStarted.Eagerly, emptyMap())
+        .stateIn(coroutineScope, SharingStarted.Lazily, emptyMap())
+
+    private val openedBuildFiles = combine(
+        project.fileOpenedFlow,
+        modulesByBuildFile.map { it.keys }
+    ) { openedFiles, buildFiles ->
+        openedFiles.filter { it.toNioPathOrNull()?.let { it in buildFiles } ?: false }
+    }
+        .shareIn(coroutineScope, SharingStarted.Lazily, 0)
 
     init {
 
@@ -151,8 +154,18 @@ class PackageSearchProjectService(
             }
             .launchIn(coroutineScope)
 
-        IntelliJApplication.PackageSearchApplicationCachesService
-            .isOnlineFlow
+        combine(
+            openedBuildFiles.map { it.isEmpty() },
+            project.toolWindowOpenedFlow("Package Search")
+        ) { noOpenedFiles, toolWindowOpened -> noOpenedFiles || !toolWindowOpened }
+            .flatMapLatest {
+                // if the tool window is not opened and there are no opened build files,
+                // we don't need to do anything, and we turn off the isOnlineFlow
+                when {
+                    it -> IntelliJApplication.PackageSearchApplicationCachesService.isOnlineFlow
+                    else -> emptyFlow()
+                }
+            }
             .filter { it }
             .onEach { restart() }
             .retry {
@@ -161,12 +174,8 @@ class PackageSearchProjectService(
             }
             .launchIn(coroutineScope)
 
-        combine(
-            project.fileOpenedFlow,
-            modulesByBuildFile.map { it.keys }
-        ) { openedFiles, buildFiles ->
-            openedFiles.filter { it.toNioPathOrNull()?.let { it in buildFiles } ?: false }
-        }
+
+        openedBuildFiles
             .filter { it.isNotEmpty() }
             .replayOn(stableOnlyStateFlow)
             .flatMapMerge { it.asFlow() }

diff --git a/...n/kotlin/com/jetbrains/packagesearch/plugin/ui/model/packageslist/PackageListViewModel.kt b/...n/kotlin/com/jetbrains/packagesearch/plugin/ui/model/packageslist/PackageListViewModel.kt
@@ -90,7 +90,7 @@ class PackageListViewModel(
         combine(listOf(selectedModuleIdsSharedFlow.map { it.size == 1 }, isOnline)) {
             it.all { it }
         }
-            .stateIn(viewModelScope, SharingStarted.Eagerly, true)
+            .stateIn(viewModelScope, SharingStarted.WhileSubscribed(), true)
 
     private val selectedModulesFlow = combine(
         selectedModuleIdsSharedFlow,
@@ -213,7 +213,7 @@ class PackageListViewModel(
                 }
             }
             .retry()
-            .stateIn(viewModelScope, SharingStarted.Eagerly, emptyList())
+            .stateIn(viewModelScope, SharingStarted.WhileSubscribed(), emptyList())
 
     private suspend fun PackageSearchModule.Base.getSearchQuery(
         searchQuery: String,

diff --git a/.../src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt b/.../src/main/kotlin/com/jetbrains/packagesearch/plugin/utils/PackageSearchApiPackageCache.kt
@@ -68,15 +68,21 @@ class PackageSearchApiPackageCache(
 
     override suspend fun getKnownRepositories(): List<ApiRepository> {
         val cached = repositoryCache.find().singleOrNull()
-        if (cached != null && (Clock.System.now() < cached.lastUpdate + maxAge || !isOnline())) {
+        val isOnlineStatus = isOnline()
+        if (cached != null && (Clock.System.now() < cached.lastUpdate + maxAge || !isOnlineStatus)) {
             return cached.data
         }
-        return if (isOnline()) apiClient.getKnownRepositories()
-            .also {
-                repositoryCache.removeAll()
-                repositoryCache.insert(ApiRepositoryCacheEntry(it))
-            }
-        else emptyList()
+        return when {
+            isOnlineStatus -> runCatching { apiClient.getKnownRepositories() }
+                .suspendSafe()
+                .onSuccess {
+                    repositoryCache.removeAll()
+                    repositoryCache.insert(ApiRepositoryCacheEntry(it))
+                }
+                .getOrDefault(cached?.data ?: emptyList())
+
+            else -> emptyList()
+        }
     }
 
     private suspend fun getPackages(
@@ -127,26 +133,12 @@ class PackageSearchApiPackageCache(
                     .suspendSafe()
                     .onFailure { logDebug("${this::class.qualifiedName}#getPackages", it) }
                 if (networkResults.isSuccess) {
-                    val packageEntries = networkResults.getOrThrow()
+                    val cacheEntriesFromNetwork = networkResults.getOrThrow()
                         .values
                         .map { it.asCacheEntry() }
-                    if (packageEntries.isNotEmpty()) {
-                        logDebug(contextName) { "No packages found | missingIds.size = ${missingIds.size}" }
-
-                        // remove the old entries
-                        apiPackageCache.remove(
-                            filter = NitriteFilters.Object.`in`(
-                                path = packageIdSelector,
-                                value = packageEntries.mapNotNull { it.packageId }
-                            )
-                        )
-                        logDebug(contextName) {
-                            "Removing old entries | packageEntries.size = ${packageEntries.size}"
-                        }
-                    }
                     // evaluate packages that are missing from our backend
                     val retrievedPackageIds =
-                        packageEntries.mapNotNull { if (useHashes) it.packageIdHash else it.packageId }
+                        cacheEntriesFromNetwork.mapNotNull { if (useHashes) it.packageIdHash else it.packageId }
                             .toSet()
                     val unknownPackages = missingIds.minus(retrievedPackageIds)
                         .map { id ->
@@ -159,8 +151,22 @@ class PackageSearchApiPackageCache(
                         "New unknown packages | unknownPackages.size = ${unknownPackages.size}"
                     }
                     // insert the new entries
-                    val toInsert = packageEntries + unknownPackages
-                    if (toInsert.isNotEmpty()) apiPackageCache.insert(toInsert)
+                    val toInsert = cacheEntriesFromNetwork + unknownPackages
+                    if (toInsert.isNotEmpty()) {
+                        toInsert.forEach { insert ->
+                            apiPackageCache.update(
+                                filter = NitriteFilters.Object.eq(
+                                    path = packageIdSelector,
+                                    value = when {
+                                        useHashes -> insert.packageIdHash
+                                        else -> insert.packageId
+                                    }
+                                ),
+                                update = insert,
+                                upsert = true
+                            )
+                        }
+                    }
                 }
                 val networkResultsData = networkResults.getOrDefault(emptyMap())
                 logDebug(contextName) {
+19 −0		http/client/src/commonMain/kotlin/org/jetbrains/packagesearch/api/v3/http/PackageSearchApi.kt
+36 −40		http/client/src/commonMain/kotlin/org/jetbrains/packagesearch/api/v3/http/PackageSearchApiClient.kt
+26 −1		http/client/src/commonMain/kotlin/org/jetbrains/packagesearch/api/v3/http/Utils.kt