Skip to content

Commit

Permalink
fix: add chunked REST garden loading + improve garden preloading
Browse files Browse the repository at this point in the history 
Previously preloading was still dependent on isFetchingGarden being false
  • Loading branch information
@th0rgall
th0rgall committed Mar 14, 2024
1 parent 43bf471 commit cb894a1
Show file tree
Hide file tree
Showing 4 changed files with 116 additions and 120 deletions.
159 changes: 76 additions & 83 deletions src/lib/api/garden.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,13 @@ import type { User } from '$lib/models/User';
import { CAMPSITES } from './collections';
import {
collection,
query,
where,
getDocs,
doc,
setDoc,
updateDoc,
getDocFromCache,
getDocFromServer,
getDoc,
CollectionReference,
limit,
startAfter
CollectionReference
} from 'firebase/firestore';
import { getUser } from '$lib/stores/auth';
import { isUploading, uploadProgress, allGardens, isFetchingGardens } from '$lib/stores/garden';
Expand Down Expand Up @@ -119,13 +114,12 @@ function mapRestToGarden(doc: RESTGardenDoc): Garden {
}

export const getAllListedGardens = async () => {
const CHUNK_SIZE = 5000;
const CHUNK_SIZE = 1500;
// To prevent endless loops in case of unexpected problems or bugs
// Note: this leads to the loop breaking once this number of gardens is reached!
const LOOP_LIMIT_ITEMS = 100000;

console.log('starting to fetch all gardens...');

console.log('Starting to fetch all gardens...');
isFetchingGardens.set(true);

let appCheckTokenResponse;
Expand All @@ -136,83 +130,82 @@ export const getAllListedGardens = async () => {
return;
}

// Include the App Check token with requests to your server.
const apiResponse = (await fetch(
`https://firestore.googleapis.com/v1/projects/${
import.meta.env.VITE_FIREBASE_PROJECT_ID
}/databases/(default)/documents:runQuery`,
{
headers: {
'X-Firebase-AppCheck': appCheckTokenResponse.token
},
method: 'POST',
body: JSON.stringify({
structuredQuery: {
from: [
{
collectionId: 'campsites',
allDescendants: false
}
],
where: {
fieldFilter: {
field: {
fieldPath: 'listed'
},
op: 'EQUAL',
value: {
booleanValue: true
let startAfterDocRef = null;
let iteration = 1;
do {
iteration++;

// Query the chunk of gardens using the REST api
const gardensChunkResponse = (await fetch(
`https://firestore.googleapis.com/v1/projects/${
import.meta.env.VITE_FIREBASE_PROJECT_ID
}/databases/(default)/documents:runQuery`,
{
headers: {
'X-Firebase-AppCheck': appCheckTokenResponse.token
},
method: 'POST',
body: JSON.stringify({
structuredQuery: {
from: [
{
collectionId: 'campsites',
allDescendants: false
}
],
where: {
fieldFilter: {
field: {
fieldPath: 'listed'
},
op: 'EQUAL',
value: {
booleanValue: true
}
}
}
},
limit: CHUNK_SIZE,
// https://stackoverflow.com/a/71812269/4973029
orderBy: [
{
direction: 'ASCENDING',
field: { fieldPath: '__name__' }
}
],
...(startAfterDocRef
? {
startAt: {
before: false,
values: [{ referenceValue: startAfterDocRef }]
}
}
: {})
}
// limit: 100
}
})
})
}
).then((r) => r.json())) as RESTGardenDoc[];

// Query the chunk of gardens
if (gardensChunkResponse.length === CHUNK_SIZE) {
// If a full chunk was fetched, there might be more gardens to fetch
startAfterDocRef = gardensChunkResponse[gardensChunkResponse.length - 1].document.name;
} else {
// If the chunk was not full, there are no more gardens to fetch
startAfterDocRef = null;
}
).then((r) => r.json())) as { document: RESTGardenDoc }[];

const gardens = apiResponse.map(mapRestToGarden);
// NOTE: not doing update anymore
allGardens.set(gardens);

// let startAfterDoc = null;
// let iteration = 1;
// do {
// iteration++;
// // Query the chunk of gardens
// const q = query.apply(null, [
// collection(db(), CAMPSITES),
// where('listed', '==', true),
// limit(CHUNK_SIZE),
// ...(startAfterDoc ? [startAfter(startAfterDoc)] : [])
// ]);
// const querySnapshot = await getDocs(q);

// if (querySnapshot.size === CHUNK_SIZE) {
// // If a full chunk was fetched, there might be more gardens to fetch
// startAfterDoc = querySnapshot.docs[querySnapshot.docs.length - 1];
// } else {
// // If the chunk was not full, there are no more gardens to fetch
// startAfterDoc = null;
// }

// // Merge the map with the existing gardens
// allGardens.update((existingGardens) => {
// querySnapshot.forEach((doc) => {
// const data = doc.data();
// existingGardens.push({
// id: doc.id,
// ...data
// });
// });
// return existingGardens;
// });

// // Wait 1000ms seconds before fetching the next chunk
// // await new Promise<void>((resolve) => setTimeout(() => resolve()));
// } while (startAfterDoc != null && iteration < LOOP_LIMIT_ITEMS / CHUNK_SIZE);

// console.log('Fetched all gardens!');

// Merge the map with the existing gardens, "in place"
allGardens.update((existingGardens) => {
// Merge the fetched gardens with the existing ones; without creating a new array in memory
// (attempt to reduce memory usage)
gardensChunkResponse.forEach((restDoc) => {
existingGardens.push(mapRestToGarden(restDoc));
});
return existingGardens;
});
} while (startAfterDocRef != null && iteration < LOOP_LIMIT_ITEMS / CHUNK_SIZE);

console.log('Fetched all gardens');

isFetchingGardens.set(false);
return get(allGardens);
Expand Down
2 changes: 1 addition & 1 deletion src/lib/components/Garden/GardenDrawer.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@
}
.garden-title .button-container {
/* Override the 100% width that causes
/* Override the 100% width that causes
the paragraph on the left to collapse on desktop drawers. */
width: auto;
margin-left: 1rem;
Expand Down
6 changes: 3 additions & 3 deletions src/lib/stores/garden.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,11 @@ import type { Garden } from '../types/Garden';

export const isUploading = writable(false);
export const uploadProgress = writable(0);
export const isFetchingGardens = writable(true);
export const isFetchingGardens = writable(false);
export const allGardens: Writable<Garden[]> = writable([]);

export const addToAllGardens = async (garden: Garden) => {
// TODO: length 0 doesn't strictly mean that the gardens are not fetched
if (get(allGardens).length === 0) {
if (get(allGardens).length === 0 && !get(isFetchingGardens)) {
isFetchingGardens.set(true);
try {
await getAllListedGardens();
Expand All @@ -19,6 +18,7 @@ export const addToAllGardens = async (garden: Garden) => {
}
isFetchingGardens.set(false);
} else {
// Update the the specific garden in the local store of gardens
allGardens.update((gardens) => {
const index = gardens.findIndex((g) => g.id === garden.id);
if (index) {
Expand Down
69 changes: 36 additions & 33 deletions src/routes/explore/+layout.svelte
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,12 +86,27 @@
let zoom = isShowingGarden ? ZOOM_LEVELS.ROAD : ZOOM_LEVELS.WESTERN_EUROPE;
$: applyZoom = isShowingGarden ? true : false;
// Garden to preload when we are loading the app on its permalink URL
let preloadedGarden: Garden | null = null;
/**
* @type {Garden | null}
*/
$: selectedGarden = $isFetchingGardens
? null
: $allGardens.find((g) => g.id === $page.params.gardenId) ?? null;
let selectedGarden = null;
// Select the preloaded garden if it matches the current URL, but only if it was not selected yet.
$: if (
preloadedGarden?.id === $page.params.gardenId &&
selectedGarden?.id !== preloadedGarden?.id
) {
selectedGarden = preloadedGarden;
}
// Select a garden when the URL changes, but only if all gardens are loaded and the garden is not selected yet.
$: if (!isEmpty($allGardens) && $page.params.gardenId !== selectedGarden?.id) {
selectedGarden = $allGardens.find((g) => g.id === $page.params.gardenId) ?? null;
}
// This variable controls the location of the map.
// Don't make it reactive based on its params, so that it can be imperatively controlled.
Expand All @@ -112,9 +127,13 @@
const selectGarden = (garden) => {
const newSelectedId = garden.id;
const newGarden = $allGardens.find((g) => g.id === newSelectedId);
setMapToGardenLocation(newGarden);
applyZoom = false; // zoom level is not programatically changed when exploring a garden
goto(`${routes.MAP}/garden/${newSelectedId}`);
if (newGarden) {
setMapToGardenLocation(newGarden);
applyZoom = false; // zoom level is not programatically changed when exploring a garden
goto(`${routes.MAP}/garden/${newSelectedId}`);
} else {
console.warn(`Failed garden navigation to ${newSelectedId}`);
}
};
const goToPlace = (event) => {
Expand All @@ -139,25 +158,22 @@
// LIFECYCLE HOOKS
onMount(async () => {
// If the gardens didn't load yet
if ($allGardens.length === 0) {
// If we're loading the page of a garden, load that one immediately before all other gardens
if (isShowingGarden) {
const garden = await getGarden($page.params.gardenId);
if (garden) {
$allGardens = [garden];
setMapToGardenLocation(garden);
}
const gardensAreEmpty = $allGardens.length === 0;
if (gardensAreEmpty && !$isFetchingGardens && isShowingGarden) {
// If we're loading the page of a garden, load that one immediately *before* all other gardens
preloadedGarden = await getGarden($page.params.gardenId);
if (preloadedGarden) {
setMapToGardenLocation(preloadedGarden);
}
}
// Fetch all gardens
try {
await getAllListedGardens();
} catch (ex) {
// Fetch all gardens if they are not loaded yet, every time the map opens
if (gardensAreEmpty && !$isFetchingGardens) {
await getAllListedGardens().catch((ex) => {
console.error(ex);
fetchError = 'Error' + ex;
isFetchingGardens.set(false);
}
});
}
});
Expand Down Expand Up @@ -215,10 +231,6 @@
<FileTrails />
<TrainconnectionsLayer />
<ZoomRestrictionNotice />
<div class="garden-debug">
{$allGardens.length} gardens
{fetchError}
</div>
</Map>
<LayersAndTools
bind:showHiking
Expand All @@ -243,15 +255,6 @@
</div>

<style>
.garden-debug {
font-size: 2rem;
position: absolute;
bottom: 50%;
right: 0;
background: white;
padding: 0.5rem;
}
.map-section {
width: 100%;
height: 100%;
Expand Down

0 comments on commit cb894a1

Please sign in to comment.