Fixed writing duplicate keys when importing

Yooooomi · Nov 21, 2023 · aece001 · aece001
1 parent b1a62b7
commit aece001
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 4 deletions.
diff --git a/server/src/spotify/dbTools.ts b/server/src/spotify/dbTools.ts
@@ -4,7 +4,7 @@ import { SpotifyAlbum, Album } from '../database/schemas/album';
 import { SpotifyArtist, Artist } from '../database/schemas/artist';
 import { SpotifyTrack, Track } from '../database/schemas/track';
 import { logger } from '../tools/logger';
-import { minOfArray, retryPromise } from '../tools/misc';
+import { minOfArray, retryPromise, uniqBy } from '../tools/misc';
 import { SpotifyAPI } from '../tools/apis/spotifyApi';
 import {
   addTrackIdsToUser,
@@ -196,13 +196,13 @@ export async function storeTrackAlbumArtist({
   artists?: Artist[];
 }) {
   if (tracks) {
-    await TrackModel.create(tracks);
+    await TrackModel.create(uniqBy(tracks, item => item.id));
   }
   if (albums) {
-    await AlbumModel.create(albums);
+    await AlbumModel.create(uniqBy(albums, item => item.id));
   }
   if (artists) {
-    await ArtistModel.create(artists);
+    await ArtistModel.create(uniqBy(artists, item => item.id));
   }
 }
 

diff --git a/server/src/tools/misc.ts b/server/src/tools/misc.ts
@@ -336,3 +336,19 @@ export function chunk<T>(array: T[], chunkSize: number) {
   }
   return chunks;
 }
+
+export function uniqBy<T>(source: T[], getId: (item: T) => string): T[] {
+  const final: T[] = [];
+  const alreadyAdded = new Set<string>();
+
+  for (const item of source) {
+    const itemId = getId(item);
+    if (alreadyAdded.has(itemId)) {
+      continue;
+    }
+    alreadyAdded.add(itemId);
+    final.push(item);
+  }
+
+  return final;
+}