From 5b2f8dbf862597eab7aeb5dc96c5d79ed74f6d4a Mon Sep 17 00:00:00 2001 From: Antoine Rousseau Date: Mon, 26 Aug 2019 13:09:52 +0200 Subject: [PATCH] handle videos --- README.md | 5 +++- start.js | 78 ++++++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index fcc84cb..a98bb9c 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,10 @@ And create a `.env` from `example.env`: ./start.js +By default, it will process all photos and videos. If you want only photos or only videos, use the `MEDIA` environment variable. Set it to `photos` or `videos`, e.g.: + + MEDIA=photos ./start.js + ## Daemonize You can use [PM2](https://github.com/Unitech/pm2) @@ -24,5 +28,4 @@ You can use [PM2](https://github.com/Unitech/pm2) ## Limitations -- This script does not handle videos - If your Google storage is limited and you hit the limit, the Google API will start to return "Bad Request". You must then either by more storage, or go to your [Google Photos settings](https://photos.google.com/settings), choose "High Quality" and click "Recover storage". This will convert your uploads to [16 Megapixels compressed photos](https://support.google.com/photos/answer/6220791), which the API cannot do on the fly. Also, you can only convert once per day. diff --git a/start.js b/start.js index 1c5bc88..9efca20 100755 --- a/start.js +++ b/start.js @@ -6,6 +6,10 @@ const googleConnect = require("./google") const { ALBUMS_PATH } = require("./constants") const { log, logError, readJson, writeJson, fileExists, mkdir } = require("./utils") +const media = process.env.MEDIA || "all" // "photos" or "videos" +const extras = "url_o,media,path_alias,original_format" +const per_page = 500 // max + const getAlbumPath = (id) => `${ALBUMS_PATH}/${id}.json` if (!fileExists(ALBUMS_PATH)) { @@ -29,32 +33,48 @@ const main = async () => { id: "NotInSet", }) - const albums_cache = {} + const memory = {} photosets.forEach((set) => { const path = getAlbumPath(set.id) if (fileExists(path)) { - albums_cache[set.id] = readJson(path) + memory[set.id] = readJson(path) } else { - albums_cache[set.id] = { - title: set.title && set.title._content, + memory[set.id] = { + title: set.id === "NotInSet" ? "Not in a set" : set.title._content, flickr_set: set.id, google_album: null, num_photos: set.photos, + num_videos: set.videos, + total: set.photos + set.videos, + uploaded_photos: 0, + uploaded_videos: 0, done: [], } - writeJson(path, albums_cache[set.id]) + writeJson(path, memory[set.id]) } }) for (let i = 0; i < photosets.length; i++) { const photoset_id = photosets[i].id + if (photoset_id !== "NotInSet") { + continue + } const path = getAlbumPath(photoset_id) - const data = albums_cache[photoset_id] - if (data.done.length === data.num_photos) { + const data = memory[photoset_id] + + if (media === "photos" && data.num_photos === data.uploaded_photos) { + continue + } + if (media === "videos" && data.num_videos === data.uploaded_videos) { + continue + } + if (media === "all" && data.num_photos + data.num_videos === data.uploaded_photos + data.uploaded_videos) { continue } + log(`Fetching ${media} in "${data.title}" photoset; Flickr id: ${photoset_id}`) + // FOR EACH PHOTOSET, RETRIEVE PHOTOS let photoset @@ -63,14 +83,27 @@ const main = async () => { page++ if (photoset_id === "NotInSet") { const { body } = await flickr.photos.getNotInSet({ - media: "photos", - extras: "url_o", + media, + extras, + per_page, page, }) photoset = body.photos - photoset.title = "Photos not in a set" - if (!data.num_photos) { - data.num_photos = Number(photoset.total) + const totalNotInSet = Number(photoset.total) + + if (!totalNotInSet) { + continue + } + if (media === "photos" && !data.num_photos) { + data.num_photos = totalNotInSet + writeJson(path, data) + } + if (media === "videos" && !data.num_videos) { + data.num_videos = totalNotInSet + writeJson(path, data) + } + if (media === "all" && !data.total) { + data.total = totalNotInSet writeJson(path, data) } } else { @@ -78,21 +111,22 @@ const main = async () => { // https://www.flickr.com/services/api/flickr.photosets.getPhotos.html photoset_id, user_id, - media: "photos", - extras: "url_o", + media, + extras, + per_page, page, }) photoset = body.photoset } - const { title, photo: photos, pages, total } = photoset + const { photo: photos, pages, total } = photoset - log(`Processing "${title}" set (${total} photos); Flickr id: ${photoset_id}; page ${page}/${pages}`) + log(`Processing page ${page}/${pages} (${photos.length}/${total} items)`) if (!data.google_album) { const albumRequest = { album: { - title, + title: data.title, }, } const { json: album } = await post("albums", albumRequest) @@ -107,9 +141,16 @@ const main = async () => { continue } + let url + if (photo.media === "video") { + url = `https://www.flickr.com/photos/${photo.pathalias}/${photo.id}/play/orig/${photo.originalsecret}/` + } else { + url = photo.url_o + } + // FOR EACH PHOTO, UPLOAD TO GOOGLE PHOTOS - const uploadToken = await stream(photo.url_o, `flickr_${photo.id}.jpg`) + const uploadToken = await stream(url, `flickr_${photo.id}.jpg`) const media = { // https://developers.google.com/photos/library/reference/rest/v1/mediaItems/batchCreate @@ -130,6 +171,7 @@ const main = async () => { if (status === 200) { data.done.push(photo.id) + data[`uploaded_${photo.media}s`]++ writeJson(path, data) log("Created media item:", results[0].mediaItem.description || "(no description)") } else {