diff --git a/commands/importer.js b/commands/importer.js index 1115922..877e45e 100644 --- a/commands/importer.js +++ b/commands/importer.js @@ -9,6 +9,7 @@ const logger = require('./../src/logger') const fetch = require('node-fetch') const FlotiqApi = require("./../src/flotiq-api"); const config = require("./../src/configuration/config"); +const {mediaImporter} = require("./../src/media"); exports.command = 'import' exports.description = 'Import flotiq entities from JSON structure' @@ -205,8 +206,7 @@ async function importer(directory, flotiqApiUrl, flotiqApiKey, skipDefinitions, `${remoteCtd ? 'Updating' : 'Persisting'} contentTypeDefinition ${contentTypeDefinition.name}` ) - // console.log('new ctd', contentTypeDefinition) - + contentTypeDefinition.featuredImage = []; const response = await fetch(uri, { method, body: JSON.stringify(contentTypeDefinition), @@ -438,6 +438,11 @@ async function importer(directory, flotiqApiUrl, flotiqApiKey, skipDefinitions, if (contentTypeDefinition.name === '_webhooks') { logger.info(`Persisting ${contentTypeDefinition.name} (${ContentObjects[contentTypeDefinition.name].length} items)`); + if (disableWebhooks) { + ContentObjects[contentTypeDefinition.name].map(webhook => { + webhook.enabled = false; + }); + } await flotiqApi .persistContentObjectBatch( contentTypeDefinition.name, @@ -448,21 +453,21 @@ async function importer(directory, flotiqApiUrl, flotiqApiKey, skipDefinitions, } } async function handler(argv) { - if (!argv.directory || !argv.flotiqApiKey) { + let directory = argv.directory; + if (!directory || !argv.flotiqApiKey) { console.error(`Usage: ${__filename} `) return false; } try { - await fs.stat(path.resolve(argv.directory)) - + await fs.stat(path.resolve(directory)) } catch (e) { - logger.error(`Cannot open import dir ${argv.directory}`) + logger.error(`Cannot open import dir ${directory}`) return false; } await importer( - argv.directory, + directory, `${config.apiUrl}/api/v1`, argv.flotiqApiKey, false, @@ -470,7 +475,13 @@ async function handler(argv) { true, true, false - ) + ); + await mediaImporter( + directory, + `${config.apiUrl}/api/v1`, + argv.flotiqApiKey + ); + } module.exports = { diff --git a/src/media.js b/src/media.js new file mode 100644 index 0000000..ddcdf7a --- /dev/null +++ b/src/media.js @@ -0,0 +1,140 @@ +const axios = require('axios'); +const fs = require('fs/promises') +const path = require('path') +const traverse = require('traverse') +const logger = require('./logger') +const FlotiqApi = require("./flotiq-api"); +const { Blob } = require('buffer'); + +async function mediaImporter (directory, flotiqApiUrl, flotiqApiKey, batchSize = 100, checkIfMediaUsed = true) { + logger.info(`Start import media files!!!!!!!!!!!!`) + const flotiqApi = new FlotiqApi(flotiqApiUrl, flotiqApiKey, { + batchSize: batchSize, + }); + + const mediaApi = axios.create({ + baseURL: `${(new URL(flotiqApiUrl)).origin}/api/media`, + timeout: flotiqApi.timeout, + headers: flotiqApi.headers, + }); + + const flotiqDefinitions = await flotiqApi.fetchContentTypeDefs() + + const mediaRelationships = flotiqDefinitions.filter(definition => { + return traverse(definition).reduce((acc, node) => { + if(node?.inputType === "datasource" && node.validation.relationContenttype === '_media' + ) { + return true; + } + return acc; + }, false) + }).map(def => def.name) + + const mediaRelationshipContentObjects = (await Promise.all(mediaRelationships.map(r => flotiqApi.fetchContentObjects(r)))).flat() + + const contentObjects = await fs + .readFile(`${directory}/InternalContentTypeMedia/contentObjectMedia.json`, 'utf-8') + .then(f => f.split('\n')) + .then(f => f.filter(el => el !== '')) + .then(f => f.map(JSON.parse)) + .then(f => f.flat()) + + const missingFiles = [] + + for (const mediaFile of contentObjects) { + const mediaFileUrl = `${(new URL(flotiqApiUrl)).origin}${mediaFile.url}` + const response = await fetch(mediaFileUrl) + + if (response.status === 404) { + missingFiles.push(mediaFile) + } + } + + const replacements = []; + + logger.info(`Will import ${missingFiles.length} missing media file(s)`) + + for (const file of missingFiles) { + let isUsed = false; + + for (const relatedContentObject of mediaRelationshipContentObjects) { + isUsed = isUsed || traverse(relatedContentObject).reduce(function (acc, node) { + if(this.key === 'dataUrl') { + const [,,,, ctd, id ] = node.split('/') + if (ctd === '_media' && id === file.id) { + return true; + } + } + return acc; + }, false) + } + + if (checkIfMediaUsed) { + if (!isUsed) { + continue; + } + } + + const buffer = await fs.readFile(`${directory}/InternalContentTypeMedia/${file.id}.${file.extension}`) + + const form = new FormData(); + + const blob = new Blob([buffer], { + type: file.mimeType + }); + + form.append('type', 'file'); + form.append('save', 1); + form.append('file', blob, file.fileName); + + const mediaEntity = await mediaApi + .post('', form, { + headers: { + 'Content-Type': 'multipart/form-data', + }, + }) + .then(res => res.data) + + replacements.push([file, mediaEntity]) + } + + logger.info('Will replace media in content objects') + + for (const relatedContentObject of mediaRelationshipContentObjects) { + + const shouldUpdate = traverse(relatedContentObject).reduce(function (acc, node) { + if(this.key === 'dataUrl') { + const [,,,, ctd, id ] = node.split('/') + if (ctd === '_media') { + let haveReplacement = false; + for (const [ originalFile, replacementFile ] of replacements) { + if (id === originalFile.id) { + this.update(`/api/v1/content/${ctd}/${replacementFile.id}`) + haveReplacement = true; + } + } + return acc || haveReplacement; + } + } + return acc; + }, false) + + if (shouldUpdate) { + logger.info(`Replacing ${relatedContentObject.id}`) + const response = await flotiqApi.middleware.put( + `/content/${relatedContentObject.internal.contentType}/${relatedContentObject.id}`, relatedContentObject) + if (response.status === 200) { + logger.info(`Updated content object ${relatedContentObject.internal.contentType}/${relatedContentObject.id}`) + } + } + } + + + logger.info(`Will delete ${missingFiles.length} broken media file(s)`) + + for (const file of missingFiles) { + await flotiqApi.middleware.delete(`/content/_media/${file.id}`).catch(() => {}) + } +} + +module.exports = {mediaImporter}