From 64ccdfaa40e1b0907bce7dabadc442b8d0d1a022 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 25 Sep 2023 15:34:51 +0200
Subject: [PATCH 01/14] Make the data key in VirtualList configurable

---
 client/src/components/History/Layout/ListingLayout.vue | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/src/components/History/Layout/ListingLayout.vue b/client/src/components/History/Layout/ListingLayout.vue
index 3c38b274fe6f..b0378191f113 100644
--- a/client/src/components/History/Layout/ListingLayout.vue
+++ b/client/src/components/History/Layout/ListingLayout.vue
@@ -4,7 +4,7 @@
             ref="listing"
             class="listing"
             role="list"
-            data-key="id"
+            :data-key="dataKey"
             :offset="offset"
             :data-sources="items"
             :data-component="{}"
@@ -32,6 +32,7 @@ export default {
         VirtualList,
     },
     props: {
+        dataKey: { type: String, default: "id" },
         offset: { type: Number, default: 0 },
         loading: { type: Boolean, default: false },
         items: { type: Array, default: null },

From 19e674ab07694c564b27e40c61344c42c99abc0e Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 25 Sep 2023 15:36:12 +0200
Subject: [PATCH 02/14] Set data key for DCEs to the `element_index`

---
 .../History/CurrentCollection/CollectionPanel.vue           | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/client/src/components/History/CurrentCollection/CollectionPanel.vue b/client/src/components/History/CurrentCollection/CollectionPanel.vue
index c5d24b753815..2d0b88f5ae49 100644
--- a/client/src/components/History/CurrentCollection/CollectionPanel.vue
+++ b/client/src/components/History/CurrentCollection/CollectionPanel.vue
@@ -115,7 +115,11 @@ watch(
             </section>
             <section class="position-relative flex-grow-1 scroller">
                 <div>
-                    <ListingLayout :items="collectionElements" :loading="loading" @scroll="onScroll">
+                    <ListingLayout
+                        data-key="element_index"
+                        :items="collectionElements"
+                        :loading="loading"
+                        @scroll="onScroll">
                         <template v-slot:item="{ item }">
                             <ContentItem
                                 :id="item.element_index + 1"

From cc202ec542d27feb65d7a91a830531bb1a6f0f2a Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 25 Sep 2023 15:37:29 +0200
Subject: [PATCH 03/14] Create placeholder elements before fetching

Since we know the expected total number of elements we can let the virtual scroller know it by creating the expected number of items beforehand and then update them as we fetch the details.
---
 client/src/stores/collectionElementsStore.ts | 84 ++++++++++++++++----
 1 file changed, 68 insertions(+), 16 deletions(-)

diff --git a/client/src/stores/collectionElementsStore.ts b/client/src/stores/collectionElementsStore.ts
index da14fddf97d0..0b70ba8c5576 100644
--- a/client/src/stores/collectionElementsStore.ts
+++ b/client/src/stores/collectionElementsStore.ts
@@ -4,10 +4,22 @@ import Vue, { computed, ref } from "vue";
 import { CollectionEntry, DCESummary, HDCASummary, HistoryContentItemBase, isHDCA } from "./services";
 import * as Service from "./services/datasetCollection.service";
 
+/**
+ * Represents a dataset collection element that has not been fetched yet.
+ */
+interface DCEPlaceholder extends DCESummary {
+    placeholder: true;
+    fetching: boolean;
+}
+
+type DCEEntry = DCEPlaceholder | DCESummary;
+
+const PLACEHOLDER_TEXT = "Loading...";
+
 export const useCollectionElementsStore = defineStore("collectionElementsStore", () => {
     const storedCollections = ref<{ [key: string]: HDCASummary }>({});
     const loadingCollectionElements = ref<{ [key: string]: boolean }>({});
-    const storedCollectionElements = ref<{ [key: string]: DCESummary[] }>({});
+    const storedCollectionElements = ref<{ [key: string]: DCEEntry[] }>({});
 
     /**
      * Returns a key that can be used to store or retrieve the elements of a collection in the store.
@@ -23,9 +35,10 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
 
     const getCollectionElements = computed(() => {
         return (collection: CollectionEntry, offset = 0, limit = 50) => {
-            const elements = storedCollectionElements.value[getCollectionKey(collection)] ?? [];
-            fetchMissingElements({ collection, offset, limit });
-            return elements ?? null;
+            const storedElements =
+                storedCollectionElements.value[getCollectionKey(collection)] ?? initWithPlaceholderElements(collection);
+            fetchMissingElements({ collection, storedElements, offset, limit });
+            return storedElements;
         };
     });
 
@@ -35,30 +48,40 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
         };
     });
 
-    async function fetchMissingElements(params: { collection: CollectionEntry; offset: number; limit: number }) {
+    async function fetchMissingElements(params: {
+        collection: CollectionEntry;
+        storedElements: DCEEntry[];
+        offset: number;
+        limit: number;
+    }) {
         const collectionKey = getCollectionKey(params.collection);
         try {
-            const maxElementCountInCollection = params.collection.element_count ?? 0;
-            const storedElements = storedCollectionElements.value[collectionKey] ?? [];
-            // Collections are immutable, so there is no need to fetch elements if the range we want is already stored
-            if (params.offset + params.limit <= storedElements.length) {
-                return;
-            }
-            // If we already have items at the offset, we can start fetching from the next offset
-            params.offset = storedElements.length;
+            // We should fetch only missing (placeholder) elements from the range
+            const firstMissingIndex = params.storedElements
+                .slice(params.offset, params.offset + params.limit)
+                .findIndex((element) => "placeholder" in element && element.fetching === false);
 
-            if (params.offset >= maxElementCountInCollection) {
+            if (firstMissingIndex === -1) {
+                // All elements in the range are already stored
                 return;
             }
+            // Adjust the offset to the first missing element
+            params.offset += firstMissingIndex;
 
             Vue.set(loadingCollectionElements.value, collectionKey, true);
+            // Mark all elements in the range as fetching
+            params.storedElements
+                .slice(params.offset, params.offset + params.limit)
+                .forEach((element) => "placeholder" in element && (element.fetching = true));
+
             const fetchedElements = await Service.fetchElementsFromCollection({
                 entry: params.collection,
                 offset: params.offset,
                 limit: params.limit,
             });
-            const updatedElements = [...storedElements, ...fetchedElements];
-            Vue.set(storedCollectionElements.value, collectionKey, updatedElements);
+            // Update only the elements that were fetched
+            params.storedElements.splice(params.offset, fetchedElements.length, ...fetchedElements);
+            Vue.set(storedCollectionElements.value, collectionKey, params.storedElements);
         } finally {
             Vue.delete(loadingCollectionElements.value, collectionKey);
         }
@@ -97,6 +120,35 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
         }
     }
 
+    function initWithPlaceholderElements(collection: HDCASummary): DCEPlaceholder[] {
+        const totalElements = collection.element_count ?? 0;
+        const placeholderElements = new Array<DCEPlaceholder>(totalElements)
+            .fill({
+                placeholder: true,
+                fetching: false,
+                element_identifier: PLACEHOLDER_TEXT,
+                element_index: 0,
+                element_type: "hda",
+                id: "placeholder",
+                model_class: "DatasetCollectionElement",
+                object: {
+                    id: "placeholder",
+                    model_class: "HistoryDatasetAssociation",
+                    state: "ok",
+                    hda_ldda: "hda",
+                    history_id: "1",
+                    tags: [],
+                },
+            })
+            .map((placeholder, index) => {
+                return {
+                    ...placeholder,
+                    element_index: index,
+                };
+            });
+        return placeholderElements;
+    }
+
     return {
         storedCollections,
         storedCollectionElements,

From 4732153af3960f1558281b3bde88633481d798a7 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 27 Sep 2023 14:09:21 +0200
Subject: [PATCH 04/14] Adapt store test for placeholders + refactor

---
 .../stores/collectionElementsStore.test.ts    | 40 ++++++++++++++-----
 client/src/stores/collectionElementsStore.ts  | 27 +++++++------
 2 files changed, 45 insertions(+), 22 deletions(-)

diff --git a/client/src/stores/collectionElementsStore.test.ts b/client/src/stores/collectionElementsStore.test.ts
index bcbcbeb4bb98..920049eefee3 100644
--- a/client/src/stores/collectionElementsStore.test.ts
+++ b/client/src/stores/collectionElementsStore.test.ts
@@ -49,7 +49,10 @@ describe("useCollectionElementsStore", () => {
         const collection1Key = store.getCollectionKey(collection1);
         const elements = store.storedCollectionElements[collection1Key];
         expect(elements).toBeDefined();
-        expect(elements).toHaveLength(limit);
+        // The total number of elements (including placeholders) is 10, but only the first 5 are fetched (real elements)
+        expect(elements).toHaveLength(10);
+        const nonPlaceholderElements = getRealElements(elements);
+        expect(nonPlaceholderElements).toHaveLength(limit);
     });
 
     it("should not fetch collection elements if they are already in the store", async () => {
@@ -62,7 +65,14 @@ describe("useCollectionElementsStore", () => {
 
         const offset = 0;
         const limit = 5;
-        // Getting collection elements should not trigger a fetch in this case
+        // Getting collection elements should trigger a fetch
+        store.getCollectionElements(collection1, offset, limit);
+        await flushPromises();
+        expect(fetchCollectionElements).toHaveBeenCalled();
+
+        fetchCollectionElements.mockClear();
+
+        // Getting the same collection elements range should not trigger a fetch
         store.getCollectionElements(collection1, offset, limit);
         expect(store.isLoadingCollectionElements(collection1)).toEqual(false);
         expect(fetchCollectionElements).not.toHaveBeenCalled();
@@ -70,11 +80,15 @@ describe("useCollectionElementsStore", () => {
 
     it("should fetch only missing elements if the requested range is not already stored", async () => {
         const store = useCollectionElementsStore();
-        const storedCount = 3;
-        const expectedStoredElements = Array.from({ length: storedCount }, (_, i) => mockElement(collection1.id, i));
-        const collection1Key = store.getCollectionKey(collection1);
-        store.storedCollectionElements[collection1Key] = expectedStoredElements;
-        expect(store.storedCollectionElements[collection1Key]).toHaveLength(storedCount);
+
+        const initialElements = 3;
+        store.getCollectionElements(collection1, 0, initialElements);
+        await flushPromises();
+        expect(fetchCollectionElements).toHaveBeenCalled();
+        let elements = store.storedCollectionElements[collection1.id];
+        // The first call will initialize the 10 placeholders and fetch the first 3 elements out of 10
+        expect(elements).toHaveLength(10);
+        expect(getRealElements(elements)).toHaveLength(initialElements);
 
         const offset = 2;
         const limit = 5;
@@ -85,11 +99,13 @@ describe("useCollectionElementsStore", () => {
         expect(store.isLoadingCollectionElements(collection1)).toEqual(false);
         expect(fetchCollectionElements).toHaveBeenCalled();
 
-        const elements = store.storedCollectionElements[collection1Key];
+        const collection1Key = store.getCollectionKey(collection1);
+        elements = store.storedCollectionElements[collection1Key];
         expect(elements).toBeDefined();
+        expect(elements).toHaveLength(10);
         // The offset was overlapping with the stored elements, so it was increased by the number of stored elements
-        // so it fetches the next "limit" number of elements
-        expect(elements).toHaveLength(storedCount + limit);
+        // and it fetches the next "limit" number of elements
+        expect(getRealElements(elements)).toHaveLength(initialElements + limit);
     });
 });
 
@@ -155,3 +171,7 @@ function fakeCollectionElementsApiResponse(params: ApiRequest) {
         data: elements,
     };
 }
+
+function getRealElements(elements?: DCESummary[]): DCESummary[] | undefined {
+    return elements?.filter((element) => element.id !== "placeholder");
+}
diff --git a/client/src/stores/collectionElementsStore.ts b/client/src/stores/collectionElementsStore.ts
index 0b70ba8c5576..2455e43b4314 100644
--- a/client/src/stores/collectionElementsStore.ts
+++ b/client/src/stores/collectionElementsStore.ts
@@ -8,13 +8,14 @@ import * as Service from "./services/datasetCollection.service";
  * Represents a dataset collection element that has not been fetched yet.
  */
 interface DCEPlaceholder extends DCESummary {
-    placeholder: true;
+    id: "placeholder";
     fetching: boolean;
 }
 
 type DCEEntry = DCEPlaceholder | DCESummary;
 
 const PLACEHOLDER_TEXT = "Loading...";
+const FETCH_LIMIT = 50;
 
 export const useCollectionElementsStore = defineStore("collectionElementsStore", () => {
     const storedCollections = ref<{ [key: string]: HDCASummary }>({});
@@ -34,7 +35,7 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
     }
 
     const getCollectionElements = computed(() => {
-        return (collection: CollectionEntry, offset = 0, limit = 50) => {
+        return (collection: CollectionEntry, offset = 0, limit = FETCH_LIMIT) => {
             const storedElements =
                 storedCollectionElements.value[getCollectionKey(collection)] ?? initWithPlaceholderElements(collection);
             fetchMissingElements({ collection, storedElements, offset, limit });
@@ -57,23 +58,22 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
         const collectionKey = getCollectionKey(params.collection);
         try {
             // We should fetch only missing (placeholder) elements from the range
-            const firstMissingIndex = params.storedElements
+            const firstMissingIndexInRange = params.storedElements
                 .slice(params.offset, params.offset + params.limit)
-                .findIndex((element) => "placeholder" in element && element.fetching === false);
+                .findIndex((element) => isPlaceholder(element) && element.fetching === false);
 
-            if (firstMissingIndex === -1) {
-                // All elements in the range are already stored
+            if (firstMissingIndexInRange === -1) {
+                // All elements in the range are already stored or being fetched
                 return;
             }
             // Adjust the offset to the first missing element
-            params.offset += firstMissingIndex;
+            params.offset += firstMissingIndexInRange;
 
             Vue.set(loadingCollectionElements.value, collectionKey, true);
             // Mark all elements in the range as fetching
             params.storedElements
                 .slice(params.offset, params.offset + params.limit)
-                .forEach((element) => "placeholder" in element && (element.fetching = true));
-
+                .forEach((element) => isPlaceholder(element) && (element.fetching = true));
             const fetchedElements = await Service.fetchElementsFromCollection({
                 entry: params.collection,
                 offset: params.offset,
@@ -120,16 +120,19 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
         }
     }
 
-    function initWithPlaceholderElements(collection: HDCASummary): DCEPlaceholder[] {
+    function isPlaceholder(element: DCEEntry): element is DCEPlaceholder {
+        return element.id === "placeholder";
+    }
+
+    function initWithPlaceholderElements(collection: CollectionEntry): DCEPlaceholder[] {
         const totalElements = collection.element_count ?? 0;
         const placeholderElements = new Array<DCEPlaceholder>(totalElements)
             .fill({
-                placeholder: true,
+                id: "placeholder",
                 fetching: false,
                 element_identifier: PLACEHOLDER_TEXT,
                 element_index: 0,
                 element_type: "hda",
-                id: "placeholder",
                 model_class: "DatasetCollectionElement",
                 object: {
                     id: "placeholder",

From 2e96558031847e1b02e15e44aebbd7ac2337ddcf Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 27 Sep 2023 14:11:53 +0200
Subject: [PATCH 05/14] Disable item expansion on placeholders

---
 client/src/components/History/Content/ContentItem.vue | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/client/src/components/History/Content/ContentItem.vue b/client/src/components/History/Content/ContentItem.vue
index aaa646b4e054..701d58346ee8 100644
--- a/client/src/components/History/Content/ContentItem.vue
+++ b/client/src/components/History/Content/ContentItem.vue
@@ -207,6 +207,9 @@ export default {
                 visualize: `/visualizations?dataset_id=${id}`,
             };
         },
+        isPlaceholder() {
+            return this.item.id === "placeholder";
+        },
     },
     methods: {
         onKeyDown(event) {
@@ -219,6 +222,9 @@ export default {
             }
         },
         onClick() {
+            if (this.isPlaceholder) {
+                return;
+            }
             if (this.isDataset) {
                 this.$emit("update:expand-dataset", !this.expandDataset);
             } else {

From f2d2e3d7b5b2b2470ac1bedbe05ce2a96cf2e114 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 28 Sep 2023 10:41:22 +0200
Subject: [PATCH 06/14] Improve imports

---
 client/src/stores/collectionElementsStore.ts | 26 ++++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/client/src/stores/collectionElementsStore.ts b/client/src/stores/collectionElementsStore.ts
index 2455e43b4314..f47a99303be1 100644
--- a/client/src/stores/collectionElementsStore.ts
+++ b/client/src/stores/collectionElementsStore.ts
@@ -1,8 +1,8 @@
 import { defineStore } from "pinia";
-import Vue, { computed, ref } from "vue";
+import { computed, del, ref, set } from "vue";
 
 import { CollectionEntry, DCESummary, HDCASummary, HistoryContentItemBase, isHDCA } from "./services";
-import * as Service from "./services/datasetCollection.service";
+import { fetchCollectionDetails, fetchElementsFromCollection } from "./services/datasetCollection.service";
 
 /**
  * Represents a dataset collection element that has not been fetched yet.
@@ -69,27 +69,27 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
             // Adjust the offset to the first missing element
             params.offset += firstMissingIndexInRange;
 
-            Vue.set(loadingCollectionElements.value, collectionKey, true);
+            set(loadingCollectionElements.value, collectionKey, true);
             // Mark all elements in the range as fetching
             params.storedElements
                 .slice(params.offset, params.offset + params.limit)
                 .forEach((element) => isPlaceholder(element) && (element.fetching = true));
-            const fetchedElements = await Service.fetchElementsFromCollection({
+            const fetchedElements = await fetchElementsFromCollection({
                 entry: params.collection,
                 offset: params.offset,
                 limit: params.limit,
             });
             // Update only the elements that were fetched
             params.storedElements.splice(params.offset, fetchedElements.length, ...fetchedElements);
-            Vue.set(storedCollectionElements.value, collectionKey, params.storedElements);
+            set(storedCollectionElements.value, collectionKey, params.storedElements);
         } finally {
-            Vue.delete(loadingCollectionElements.value, collectionKey);
+            del(loadingCollectionElements.value, collectionKey);
         }
     }
 
     async function loadCollectionElements(collection: CollectionEntry) {
-        const elements = await Service.fetchElementsFromCollection({ entry: collection });
-        Vue.set(storedCollectionElements.value, getCollectionKey(collection), elements);
+        const elements = await fetchElementsFromCollection({ entry: collection });
+        set(storedCollectionElements.value, getCollectionKey(collection), elements);
     }
 
     function saveCollections(historyContentsPayload: HistoryContentItemBase[]) {
@@ -97,7 +97,7 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
             (entry) => entry.history_content_type === "dataset_collection"
         ) as HDCASummary[];
         for (const collection of collectionsInHistory) {
-            Vue.set<HDCASummary>(storedCollections.value, collection.id, collection);
+            set<HDCASummary>(storedCollections.value, collection.id, collection);
         }
     }
 
@@ -110,13 +110,13 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
     }
 
     async function fetchCollection(params: { id: string }) {
-        Vue.set(loadingCollectionElements.value, params.id, true);
+        set(loadingCollectionElements.value, params.id, true);
         try {
-            const collection = await Service.fetchCollectionDetails({ hdcaId: params.id });
-            Vue.set(storedCollections.value, collection.id, collection);
+            const collection = await fetchCollectionDetails({ hdcaId: params.id });
+            set(storedCollections.value, collection.id, collection);
             return collection;
         } finally {
-            Vue.delete(loadingCollectionElements.value, params.id);
+            del(loadingCollectionElements.value, params.id);
         }
     }
 

From 77b6ab8ba52da7e36667a4386974496b65aee935 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 28 Sep 2023 17:16:38 +0200
Subject: [PATCH 07/14] Simplify ContentPlaceholder

Remove unnecessary defaults and keep it to minimum.
---
 client/src/stores/collectionElementsStore.ts | 56 ++++++++------------
 1 file changed, 23 insertions(+), 33 deletions(-)

diff --git a/client/src/stores/collectionElementsStore.ts b/client/src/stores/collectionElementsStore.ts
index f47a99303be1..5d212f717304 100644
--- a/client/src/stores/collectionElementsStore.ts
+++ b/client/src/stores/collectionElementsStore.ts
@@ -5,16 +5,26 @@ import { CollectionEntry, DCESummary, HDCASummary, HistoryContentItemBase, isHDC
 import { fetchCollectionDetails, fetchElementsFromCollection } from "./services/datasetCollection.service";
 
 /**
- * Represents a dataset collection element that has not been fetched yet.
+ * Represents an element in a collection that has not been fetched yet.
  */
-interface DCEPlaceholder extends DCESummary {
-    id: "placeholder";
-    fetching: boolean;
+export interface ContentPlaceholder {
+    /**
+     * The index of the element in the containing collection.
+     *
+     * This is used to determine the offset to fetch the element from when
+     * scrolling through the collection.
+     */
+    element_index?: number;
+    /**
+     * Whether the element is currently being fetched.
+     *
+     * This is used to prevent fetching the same element multiple times.
+     */
+    fetching?: boolean;
 }
 
-type DCEEntry = DCEPlaceholder | DCESummary;
+type DCEEntry = ContentPlaceholder | DCESummary;
 
-const PLACEHOLDER_TEXT = "Loading...";
 const FETCH_LIMIT = 50;
 
 export const useCollectionElementsStore = defineStore("collectionElementsStore", () => {
@@ -60,7 +70,7 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
             // We should fetch only missing (placeholder) elements from the range
             const firstMissingIndexInRange = params.storedElements
                 .slice(params.offset, params.offset + params.limit)
-                .findIndex((element) => isPlaceholder(element) && element.fetching === false);
+                .findIndex((element) => isPlaceholder(element) && !element.fetching);
 
             if (firstMissingIndexInRange === -1) {
                 // All elements in the range are already stored or being fetched
@@ -120,35 +130,15 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
         }
     }
 
-    function isPlaceholder(element: DCEEntry): element is DCEPlaceholder {
-        return element.id === "placeholder";
+    function isPlaceholder(element: DCEEntry): element is ContentPlaceholder {
+        return "id" in element === false;
     }
 
-    function initWithPlaceholderElements(collection: CollectionEntry): DCEPlaceholder[] {
+    function initWithPlaceholderElements(collection: CollectionEntry): ContentPlaceholder[] {
         const totalElements = collection.element_count ?? 0;
-        const placeholderElements = new Array<DCEPlaceholder>(totalElements)
-            .fill({
-                id: "placeholder",
-                fetching: false,
-                element_identifier: PLACEHOLDER_TEXT,
-                element_index: 0,
-                element_type: "hda",
-                model_class: "DatasetCollectionElement",
-                object: {
-                    id: "placeholder",
-                    model_class: "HistoryDatasetAssociation",
-                    state: "ok",
-                    hda_ldda: "hda",
-                    history_id: "1",
-                    tags: [],
-                },
-            })
-            .map((placeholder, index) => {
-                return {
-                    ...placeholder,
-                    element_index: index,
-                };
-            });
+        const placeholderElements = new Array<ContentPlaceholder>(totalElements).fill({}).map((_, index) => ({
+            element_index: index,
+        }));
         return placeholderElements;
     }
 

From e086b328bb4633fd4684a93af4ed551acbe9356c Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 28 Sep 2023 17:18:18 +0200
Subject: [PATCH 08/14] Refactor ContentItem props

- Make `expandDataset` optional with default to false.
- Reorder props to put required ones first.
---
 client/src/components/History/Content/ContentItem.vue | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/src/components/History/Content/ContentItem.vue b/client/src/components/History/Content/ContentItem.vue
index 701d58346ee8..ea157caa6217 100644
--- a/client/src/components/History/Content/ContentItem.vue
+++ b/client/src/components/History/Content/ContentItem.vue
@@ -126,15 +126,15 @@ export default {
         FontAwesomeIcon,
     },
     props: {
+        id: { type: Number, required: true },
+        item: { type: Object, required: true },
+        name: { type: String, required: true },
+        expandDataset: { type: Boolean, default: false },
         writable: { type: Boolean, default: true },
-        expandDataset: { type: Boolean, required: true },
         addHighlightBtn: { type: Boolean, default: false },
         highlight: { type: String, default: null },
-        id: { type: Number, required: true },
         isDataset: { type: Boolean, default: true },
         isHistoryItem: { type: Boolean, default: false },
-        item: { type: Object, required: true },
-        name: { type: String, required: true },
         selected: { type: Boolean, default: false },
         selectable: { type: Boolean, default: false },
         filterable: { type: Boolean, default: false },

From d9f5ecf66d5c1f09266557e427fb753c369796ec Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 28 Sep 2023 17:19:53 +0200
Subject: [PATCH 09/14] Allow placeholders as ContentItems

---
 client/src/components/History/Content/ContentItem.vue      | 6 ++----
 .../History/CurrentCollection/CollectionPanel.vue          | 7 +++++++
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/client/src/components/History/Content/ContentItem.vue b/client/src/components/History/Content/ContentItem.vue
index ea157caa6217..410a1d9726ff 100644
--- a/client/src/components/History/Content/ContentItem.vue
+++ b/client/src/components/History/Content/ContentItem.vue
@@ -54,7 +54,7 @@
                     </b-badge>
                 </span>
                 <ContentOptions
-                    v-else
+                    v-else-if="!isPlaceholder"
                     :writable="writable"
                     :is-dataset="isDataset"
                     :is-deleted="item.deleted"
@@ -138,6 +138,7 @@ export default {
         selected: { type: Boolean, default: false },
         selectable: { type: Boolean, default: false },
         filterable: { type: Boolean, default: false },
+        isPlaceholder: { type: Boolean, default: false },
     },
     computed: {
         jobState() {
@@ -207,9 +208,6 @@ export default {
                 visualize: `/visualizations?dataset_id=${id}`,
             };
         },
-        isPlaceholder() {
-            return this.item.id === "placeholder";
-        },
     },
     methods: {
         onKeyDown(event) {
diff --git a/client/src/components/History/CurrentCollection/CollectionPanel.vue b/client/src/components/History/CurrentCollection/CollectionPanel.vue
index 2d0b88f5ae49..0118f52ee667 100644
--- a/client/src/components/History/CurrentCollection/CollectionPanel.vue
+++ b/client/src/components/History/CurrentCollection/CollectionPanel.vue
@@ -122,6 +122,13 @@ watch(
                         @scroll="onScroll">
                         <template v-slot:item="{ item }">
                             <ContentItem
+                                v-if="item.id === undefined"
+                                :id="item.element_index + 1"
+                                :item="item"
+                                :is-placeholder="true"
+                                name="Loading..." />
+                            <ContentItem
+                                v-else
                                 :id="item.element_index + 1"
                                 :item="item.object"
                                 :name="item.element_identifier"

From 527652a2a964247163851bf797a2d71f46697d49 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Thu, 28 Sep 2023 17:50:32 +0200
Subject: [PATCH 10/14] Adapt unit tests

---
 client/src/stores/collectionElementsStore.test.ts | 6 +++---
 client/src/stores/collectionElementsStore.ts      | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/src/stores/collectionElementsStore.test.ts b/client/src/stores/collectionElementsStore.test.ts
index 920049eefee3..8524b5a8f277 100644
--- a/client/src/stores/collectionElementsStore.test.ts
+++ b/client/src/stores/collectionElementsStore.test.ts
@@ -2,7 +2,7 @@ import flushPromises from "flush-promises";
 import { createPinia, setActivePinia } from "pinia";
 
 import { mockFetcher } from "@/schema/__mocks__";
-import { useCollectionElementsStore } from "@/stores/collectionElementsStore";
+import { DCEEntry, useCollectionElementsStore } from "@/stores/collectionElementsStore";
 import { DCESummary, HDCASummary } from "@/stores/services";
 
 jest.mock("@/schema");
@@ -172,6 +172,6 @@ function fakeCollectionElementsApiResponse(params: ApiRequest) {
     };
 }
 
-function getRealElements(elements?: DCESummary[]): DCESummary[] | undefined {
-    return elements?.filter((element) => element.id !== "placeholder");
+function getRealElements(elements?: DCEEntry[]): DCESummary[] | undefined {
+    return elements?.filter((element) => "id" in element === true) as DCESummary[];
 }
diff --git a/client/src/stores/collectionElementsStore.ts b/client/src/stores/collectionElementsStore.ts
index 5d212f717304..a86d44b32815 100644
--- a/client/src/stores/collectionElementsStore.ts
+++ b/client/src/stores/collectionElementsStore.ts
@@ -23,7 +23,7 @@ export interface ContentPlaceholder {
     fetching?: boolean;
 }
 
-type DCEEntry = ContentPlaceholder | DCESummary;
+export type DCEEntry = ContentPlaceholder | DCESummary;
 
 const FETCH_LIMIT = 50;
 

From 9c0a1037f628b8981dcb521b4aafbe2b94c98602 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Fri, 29 Sep 2023 13:16:57 +0200
Subject: [PATCH 11/14] Display spinner in placeholder

---
 client/src/components/History/Content/ContentItem.vue | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/client/src/components/History/Content/ContentItem.vue b/client/src/components/History/Content/ContentItem.vue
index 410a1d9726ff..b9ce711fd4ba 100644
--- a/client/src/components/History/Content/ContentItem.vue
+++ b/client/src/components/History/Content/ContentItem.vue
@@ -53,8 +53,11 @@
                         <icon icon="burn" /> Purged
                     </b-badge>
                 </span>
+                <span v-else-if="isPlaceholder" class="align-self-start btn-group p-1">
+                    <FontAwesomeIcon icon="spinner" spin />
+                </span>
                 <ContentOptions
-                    v-else-if="!isPlaceholder"
+                    v-else
                     :writable="writable"
                     :is-dataset="isDataset"
                     :is-deleted="item.deleted"
@@ -102,7 +105,7 @@
 
 <script>
 import { library } from "@fortawesome/fontawesome-svg-core";
-import { faArrowCircleDown, faArrowCircleUp, faCheckCircle } from "@fortawesome/free-solid-svg-icons";
+import { faArrowCircleDown, faArrowCircleUp, faCheckCircle, faSpinner } from "@fortawesome/free-solid-svg-icons";
 import { FontAwesomeIcon } from "@fortawesome/vue-fontawesome";
 import { updateContentFields } from "components/History/model/queries";
 import StatelessTags from "components/TagsMultiselect/StatelessTags";
@@ -116,7 +119,7 @@ import ContentOptions from "./ContentOptions";
 import DatasetDetails from "./Dataset/DatasetDetails";
 import { HIERARCHICAL_COLLECTION_JOB_STATES, STATES } from "./model/states";
 
-library.add(faArrowCircleUp, faArrowCircleDown, faCheckCircle);
+library.add(faArrowCircleUp, faArrowCircleDown, faCheckCircle, faSpinner);
 export default {
     components: {
         CollectionDescription,

From e388897ec788fe5aa7f74007031a616574e65fb3 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Mon, 2 Oct 2023 12:22:18 +0200
Subject: [PATCH 12/14] Add `placeholder` as special visual state

---
 client/src/components/History/Content/ContentItem.vue |  8 ++++----
 client/src/components/History/Content/model/states.ts | 11 +++++++++--
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/client/src/components/History/Content/ContentItem.vue b/client/src/components/History/Content/ContentItem.vue
index b9ce711fd4ba..a9081cc7019e 100644
--- a/client/src/components/History/Content/ContentItem.vue
+++ b/client/src/components/History/Content/ContentItem.vue
@@ -53,11 +53,8 @@
                         <icon icon="burn" /> Purged
                     </b-badge>
                 </span>
-                <span v-else-if="isPlaceholder" class="align-self-start btn-group p-1">
-                    <FontAwesomeIcon icon="spinner" spin />
-                </span>
                 <ContentOptions
-                    v-else
+                    v-if="!isPlaceholder && !item.purged"
                     :writable="writable"
                     :is-dataset="isDataset"
                     :is-deleted="item.deleted"
@@ -170,6 +167,9 @@ export default {
             return this.contentState && this.contentState.icon;
         },
         state() {
+            if (this.isPlaceholder) {
+                return "placeholder";
+            }
             if (this.item.job_state_summary) {
                 for (const state of HIERARCHICAL_COLLECTION_JOB_STATES) {
                     if (this.item.job_state_summary[state] > 0) {
diff --git a/client/src/components/History/Content/model/states.ts b/client/src/components/History/Content/model/states.ts
index 4bd87f544382..67eab750bbce 100644
--- a/client/src/components/History/Content/model/states.ts
+++ b/client/src/components/History/Content/model/states.ts
@@ -2,10 +2,10 @@ import { components } from "@/schema";
 
 type DatasetState = components["schemas"]["DatasetState"];
 // The 'failed' state is for the collection job state summary, not a dataset state.
-type State = DatasetState | "failed";
+type State = DatasetState | "failed" | "placeholder";
 
 interface StateRepresentation {
-    status: "success" | "warning" | "info" | "danger";
+    status: "success" | "warning" | "info" | "danger" | "secondary";
     text?: string;
     icon?: string;
     spin?: boolean;
@@ -97,6 +97,13 @@ export const STATES: StateMap = {
         status: "danger",
         icon: "exclamation-triangle",
     },
+    /** the dataset is not yet loaded in the UI. This state is only visual and transitional, it does not exist in the database. */
+    placeholder: {
+        status: "secondary",
+        text: "This dataset is being fetched.",
+        icon: "spinner",
+        spin: true,
+    },
 } as const satisfies StateMap;
 
 /** We want to display a single state for a dataset collection whose elements may have mixed states.

From 739193a3da8f0861feb3199911b2cac808b4ae31 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 11 Oct 2023 12:18:28 +0200
Subject: [PATCH 13/14] Refactor unit tests

---
 .../stores/collectionElementsStore.test.ts    | 73 ++++++++++---------
 1 file changed, 38 insertions(+), 35 deletions(-)

diff --git a/client/src/stores/collectionElementsStore.test.ts b/client/src/stores/collectionElementsStore.test.ts
index 8524b5a8f277..475da2cf7668 100644
--- a/client/src/stores/collectionElementsStore.test.ts
+++ b/client/src/stores/collectionElementsStore.test.ts
@@ -7,10 +7,6 @@ import { DCESummary, HDCASummary } from "@/stores/services";
 
 jest.mock("@/schema");
 
-const collection1: HDCASummary = mockCollection("1");
-const collection2: HDCASummary = mockCollection("2");
-const collections: HDCASummary[] = [collection1, collection2];
-
 describe("useCollectionElementsStore", () => {
     beforeEach(() => {
         setActivePinia(createPinia());
@@ -21,6 +17,9 @@ describe("useCollectionElementsStore", () => {
     });
 
     it("should save collections", async () => {
+        const collection1: HDCASummary = mockCollection("1");
+        const collection2: HDCASummary = mockCollection("2");
+        const collections: HDCASummary[] = [collection1, collection2];
         const store = useCollectionElementsStore();
         expect(store.storedCollections).toEqual({});
 
@@ -33,74 +32,74 @@ describe("useCollectionElementsStore", () => {
     });
 
     it("should fetch collection elements if they are not yet in the store", async () => {
+        const totalElements = 10;
+        const collection: HDCASummary = mockCollection("1", totalElements);
         const store = useCollectionElementsStore();
         expect(store.storedCollectionElements).toEqual({});
-        expect(store.isLoadingCollectionElements(collection1)).toEqual(false);
+        expect(store.isLoadingCollectionElements(collection)).toEqual(false);
 
         const offset = 0;
         const limit = 5;
-        // Getting collection elements should trigger a fetch
-        store.getCollectionElements(collection1, offset, limit);
-        expect(store.isLoadingCollectionElements(collection1)).toEqual(true);
+        // Getting collection elements should trigger a fetch and change the loading state
+        store.getCollectionElements(collection, offset, limit);
+        expect(store.isLoadingCollectionElements(collection)).toEqual(true);
         await flushPromises();
-        expect(store.isLoadingCollectionElements(collection1)).toEqual(false);
+        expect(store.isLoadingCollectionElements(collection)).toEqual(false);
         expect(fetchCollectionElements).toHaveBeenCalled();
 
-        const collection1Key = store.getCollectionKey(collection1);
-        const elements = store.storedCollectionElements[collection1Key];
+        const collectionKey = store.getCollectionKey(collection);
+        const elements = store.storedCollectionElements[collectionKey];
         expect(elements).toBeDefined();
         // The total number of elements (including placeholders) is 10, but only the first 5 are fetched (real elements)
-        expect(elements).toHaveLength(10);
+        expect(elements).toHaveLength(totalElements);
         const nonPlaceholderElements = getRealElements(elements);
         expect(nonPlaceholderElements).toHaveLength(limit);
     });
 
     it("should not fetch collection elements if they are already in the store", async () => {
+        const totalElements = 10;
+        const collection: HDCASummary = mockCollection("1", totalElements);
         const store = useCollectionElementsStore();
+        // Prefill the store with the first 5 elements
         const storedCount = 5;
-        const expectedStoredElements = Array.from({ length: storedCount }, (_, i) => mockElement(collection1.id, i));
-        const collection1Key = store.getCollectionKey(collection1);
-        store.storedCollectionElements[collection1Key] = expectedStoredElements;
-        expect(store.storedCollectionElements[collection1Key]).toHaveLength(storedCount);
+        const expectedStoredElements = Array.from({ length: storedCount }, (_, i) => mockElement(collection.id, i));
+        const collectionKey = store.getCollectionKey(collection);
+        store.storedCollectionElements[collectionKey] = expectedStoredElements;
+        expect(store.storedCollectionElements[collectionKey]).toHaveLength(storedCount);
 
         const offset = 0;
-        const limit = 5;
-        // Getting collection elements should trigger a fetch
-        store.getCollectionElements(collection1, offset, limit);
-        await flushPromises();
-        expect(fetchCollectionElements).toHaveBeenCalled();
-
-        fetchCollectionElements.mockClear();
-
+        const limit = storedCount;
         // Getting the same collection elements range should not trigger a fetch
-        store.getCollectionElements(collection1, offset, limit);
-        expect(store.isLoadingCollectionElements(collection1)).toEqual(false);
+        store.getCollectionElements(collection, offset, limit);
+        expect(store.isLoadingCollectionElements(collection)).toEqual(false);
         expect(fetchCollectionElements).not.toHaveBeenCalled();
     });
 
     it("should fetch only missing elements if the requested range is not already stored", async () => {
+        const totalElements = 10;
+        const collection: HDCASummary = mockCollection("1", totalElements);
         const store = useCollectionElementsStore();
 
         const initialElements = 3;
-        store.getCollectionElements(collection1, 0, initialElements);
+        store.getCollectionElements(collection, 0, initialElements);
         await flushPromises();
         expect(fetchCollectionElements).toHaveBeenCalled();
-        let elements = store.storedCollectionElements[collection1.id];
+        const collectionKey = store.getCollectionKey(collection);
+        let elements = store.storedCollectionElements[collectionKey];
         // The first call will initialize the 10 placeholders and fetch the first 3 elements out of 10
-        expect(elements).toHaveLength(10);
+        expect(elements).toHaveLength(totalElements);
         expect(getRealElements(elements)).toHaveLength(initialElements);
 
         const offset = 2;
         const limit = 5;
         // Getting collection elements should trigger a fetch in this case
-        store.getCollectionElements(collection1, offset, limit);
-        expect(store.isLoadingCollectionElements(collection1)).toEqual(true);
+        store.getCollectionElements(collection, offset, limit);
+        expect(store.isLoadingCollectionElements(collection)).toEqual(true);
         await flushPromises();
-        expect(store.isLoadingCollectionElements(collection1)).toEqual(false);
+        expect(store.isLoadingCollectionElements(collection)).toEqual(false);
         expect(fetchCollectionElements).toHaveBeenCalled();
 
-        const collection1Key = store.getCollectionKey(collection1);
-        elements = store.storedCollectionElements[collection1Key];
+        elements = store.storedCollectionElements[collectionKey];
         expect(elements).toBeDefined();
         expect(elements).toHaveLength(10);
         // The offset was overlapping with the stored elements, so it was increased by the number of stored elements
@@ -116,7 +115,7 @@ function mockCollection(id: string, numElements = 10): HDCASummary {
         collection_type: "list",
         populated_state: "ok",
         populated_state_message: "",
-        collection_id: id,
+        collection_id: `DC_ID_${id}`,
         name: `collection ${id}`,
         deleted: false,
         contents_url: "",
@@ -154,6 +153,7 @@ function mockElement(collectionId: string, i: number): DCESummary {
 
 interface ApiRequest {
     hdca_id: string;
+    parent_id: string;
     offset: number;
     limit: number;
 }
@@ -172,6 +172,9 @@ function fakeCollectionElementsApiResponse(params: ApiRequest) {
     };
 }
 
+/**
+ * Filter out the placeholder elements from the given array.
+ */
 function getRealElements(elements?: DCEEntry[]): DCESummary[] | undefined {
     return elements?.filter((element) => "id" in element === true) as DCESummary[];
 }

From 3b19e6542548b8691350ebad6aaa9bd88bcc8120 Mon Sep 17 00:00:00 2001
From: davelopez <46503462+davelopez@users.noreply.github.com>
Date: Wed, 11 Oct 2023 15:50:26 +0200
Subject: [PATCH 14/14] Init placeholders in loop

---
 client/src/stores/collectionElementsStore.ts | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/client/src/stores/collectionElementsStore.ts b/client/src/stores/collectionElementsStore.ts
index a86d44b32815..8f97ad319745 100644
--- a/client/src/stores/collectionElementsStore.ts
+++ b/client/src/stores/collectionElementsStore.ts
@@ -14,7 +14,7 @@ export interface ContentPlaceholder {
      * This is used to determine the offset to fetch the element from when
      * scrolling through the collection.
      */
-    element_index?: number;
+    element_index: number;
     /**
      * Whether the element is currently being fetched.
      *
@@ -136,9 +136,10 @@ export const useCollectionElementsStore = defineStore("collectionElementsStore",
 
     function initWithPlaceholderElements(collection: CollectionEntry): ContentPlaceholder[] {
         const totalElements = collection.element_count ?? 0;
-        const placeholderElements = new Array<ContentPlaceholder>(totalElements).fill({}).map((_, index) => ({
-            element_index: index,
-        }));
+        const placeholderElements = new Array<ContentPlaceholder>(totalElements);
+        for (let i = 0; i < totalElements; i++) {
+            placeholderElements[i] = { element_index: i };
+        }
         return placeholderElements;
     }