PHRAS-3928_download_async WIP ok to test (#4386)

* WIP/POC DO NOT MERGE use "pusher" to wait for export-by-email worker to tell client that export is done. nb: export worker is artificially delayed by 30s ! * WIP OK TO TEST ; DO NOT MERGE * WIP/POC DO NOT MERGE use "pusher" to wait for export-by-email worker to tell client that export is done. nb: export worker is artificially delayed by 30s ! * WIP OK TO TEST ; DO NOT MERGE * cleanup * cleanup * better conf & cleanup * fix typo * fix stamp transparency (bump imagine) fix missing cgu for one file download better cli feedback add (re)download link on cli * fix test * fix missing js feedback (when worker publish before client subscribes) * cleanup * fix "remove stamp" choice * add default conf * WIP/POC DO NOT MERGE use "pusher" to wait for export-by-email worker to tell client that export is done. nb: export worker is artificially delayed by 30s ! * WIP OK TO TEST ; DO NOT MERGE * WIP/POC DO NOT MERGE use "pusher" to wait for export-by-email worker to tell client that export is done. nb: export worker is artificially delayed by 30s ! * WIP OK TO TEST ; DO NOT MERGE * cleanup * cleanup * better conf & cleanup * fix typo * fix stamp transparency (bump imagine) fix missing cgu for one file download better cli feedback add (re)download link on cli * fix test * fix missing js feedback (when worker publish before client subscribes) * cleanup * fix "remove stamp" choice * add default conf * WIP OK TO TEST generates an excel report for async download. define some env-vars for Pusher (todo: fix entrypoint to add during install) * fix xl formating for tabs >1 add env_vars to config build * fix test
alchemy-fr · Oct 30, 2023 · 7d703b6 · 7d703b6
1 parent a6c459c
commit 7d703b6
Show file tree

Hide file tree

Showing 42 changed files with 1,673 additions and 152 deletions.
diff --git a/.env b/.env
@@ -64,6 +64,7 @@
 # - "deleteRecord"
 # - "editRecord"
 # - "exportMail"
+# - "downloadAsync"
 # - "exposeUpload"
 # - "exportFtp"
 # - "mainQueue"
@@ -105,7 +106,7 @@
 # Example with all profiles:
 # - COMPOSE_FILE=docker-compose.yml:docker-compose.datastores.yml:docker-compose.tools.yml
 # - COMPOSE_PROFILES=app,setup,gateway-classic,db,elasticsearch,redis,redis-session,rabbitmq,pma,mailhog,assetsInjest,createRecord,deleteRecord,editRecord,
-#                    exportMail,exposeUpload,exportFtp,mainQueue,populateIndex,pullAssets,recordsActions,subdefCreation,
+#                    exportMail,downloadAsync,exposeUpload,exportFtp,mainQueue,populateIndex,pullAssets,recordsActions,subdefCreation,
 #                    validationReminder,webhook,writeMetadatas,shareBasket,scheduler,elk,db-backup,phraseanet-saml-sp
 #
 
@@ -197,7 +198,10 @@ GATEWAY_FASTCGI_HTTPS=off
 # Content Security Policy (CSP)
 # security that helps to detect and mitigate certain types of attacks, including Cross-Site Scripting
 ## @run 
-GATEWAY_CSP="default-src 'self' 127.0.0.1 https://apiws.carrick-skills.com:8443 https://apiws.carrick-flow.com:8443 https://fonts.gstatic.com *.tiles.mapbox.com https://api.mapbox.com https://events.mapbox.com *.axept.io *.matomo.cloud *.newrelic.com *.nr-data.net https://www.googletagmanager.com *.google-analytics.com *.phrasea.io https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443  https://maxcdn.bootstrapcdn.com data: ; script-src 'unsafe-inline' 'unsafe-eval' 'self' https://www.gstatic.com *.alchemyasp.com *.axept.io *.matomo.cloud *.newrelic.com https://www.googletagmanager.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443  https://maxcdn.bootstrapcdn.com  data: blob: ; style-src 'self' 'unsafe-inline' https://fonts.gstatic.com https://fonts.googleapis.com https://www.google.com https://www.gstatic.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443  https://maxcdn.bootstrapcdn.com ; img-src 'self'  data: blob: *.tiles.mapbox.com https://axeptio.imgix.net *.cloudfront.net *.phrasea.io *.amazonaws.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443  https://maxcdn.bootstrapcdn.com https://www.gnu.org/graphics/ ; object-src 'self'; frame-ancestors 'self'"
+## GATEWAY_CSP="default-src 'self' 127.0.0.1 https://apiws.carrick-skills.com:8443 https://apiws.carrick-flow.com:8443 https://fonts.gstatic.com *.tiles.mapbox.com https://api.mapbox.com https://events.mapbox.com *.axept.io *.matomo.cloud *.newrelic.com *.nr-data.net https://www.googletagmanager.com *.google-analytics.com *.phrasea.io https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443  https://maxcdn.bootstrapcdn.com data: ; script-src 'unsafe-inline' 'unsafe-eval' 'self' https://www.gstatic.com *.alchemyasp.com *.axept.io *.matomo.cloud *.newrelic.com https://www.googletagmanager.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443  https://maxcdn.bootstrapcdn.com  data: blob: ; style-src 'self' 'unsafe-inline' https://fonts.gstatic.com https://fonts.googleapis.com https://www.google.com https://www.gstatic.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443  https://maxcdn.bootstrapcdn.com ; img-src 'self'  data: blob: *.tiles.mapbox.com https://axeptio.imgix.net *.cloudfront.net *.phrasea.io *.amazonaws.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443  https://maxcdn.bootstrapcdn.com https://www.gnu.org/graphics/ https://sockjs-eu.pusher.com:443 wss://ws-eu.pusher.com ; object-src 'self'; frame-ancestors 'self'"
+
+
+GATEWAY_CSP="default-src 'self' 127.0.0.1 https://sockjs-eu.pusher.com:443 wss://ws-eu.pusher.com https://apiws.carrick-skills.com:8443 https://apiws.carrick-flow.com:8443 https://fonts.gstatic.com *.tiles.mapbox.com https://api.mapbox.com https://events.mapbox.com *.axept.io *.matomo.cloud *.newrelic.com *.nr-data.net https://www.googletagmanager.com *.google-analytics.com *.phrasea.io https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 data: ;script-src 'unsafe-inline' 'unsafe-eval' 'self' https://www.gstatic.com *.alchemyasp.com *.axept.io *.matomo.cloud *.newrelic.com https://www.googletagmanager.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 ;style-src 'self' 'unsafe-inline' https://fonts.gstatic.com https://fonts.googleapis.com https://www.google.com https://www.gstatic.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443;img-src 'self' data: blob: *.tiles.mapbox.com https://axeptio.imgix.net *.cloudfront.net *.phrasea.io *.amazonaws.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 ; object-src 'self';frame-ancestors 'self'"
 
 # --- RabbitMQ settings ------------------------------------------------------------------------------------------------
 
@@ -414,6 +418,23 @@ DB_BACKUP_CRON_TIME=
 DB_BACKUP_GZIP_LEVEL=9
 
 
+# --- Pusher settings --------------------------------------------------------------------------------------
+
+# Pusher settings used when PHRASEANET_DOWNLOAD_ASYNC=true (configuration.yml: download_async / enabled=true)
+
+# key
+# @run
+PUSHER_AUTH_KEY
+
+# secret
+# @run
+PUSHER_SECRET
+
+# app_id
+# @run
+PUSHER_APP_ID
+
+
 # --- Application cache settings ---------------------------------------------------------------------------------------------------
 
 # Cache setting type can be "redis" or "arraycache"
@@ -463,6 +484,9 @@ [email protected]
 # @run
 PHRASEANET_ADMIN_ACCOUNT_PASSWORD=iJRqXU0MwbyJewQLBbra6IWHsWly
 
+# Use Pusher to enable async download.
+# @run
+PHRASEANET_DOWNLOAD_ASYNC=false
 
 
 # --- Phraseanet MySQL settings ----------------------------------------------------------------------------------------
@@ -686,6 +710,9 @@ PHRASEANET_WORKER_editRecord=2
 # @run
 PHRASEANET_WORKER_exportMail=2
 
+# @run
+PHRASEANET_WORKER_downloadAsync=2
+
 # @run
 PHRASEANET_WORKER_exposeUpload=2
 

diff --git a/Phraseanet-production-client/config/config.js b/Phraseanet-production-client/config/config.js
@@ -13,5 +13,5 @@ module.exports = {
     setupDir: _root + 'tests/setup/node.js',
     karmaConf: _root + 'config/karma.conf.js',
     // change this version when you change JS file for lazy loading
-    assetFileVersion: 95
+    assetFileVersion: 96
 };
diff --git a/Phraseanet-production-client/dist/authenticate.js b/Phraseanet-production-client/dist/authenticate.js
@@ -96,7 +96,7 @@ return /******/ (function(modules) { // webpackBootstrap
 /******/ 		if (__webpack_require__.nc) {
 /******/ 			script.setAttribute("nonce", __webpack_require__.nc);
 /******/ 		}
-/******/ 		script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".js?v=95";
+/******/ 		script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".js?v=96";
 /******/ 		var timeout = setTimeout(onScriptComplete, 120000);
 /******/ 		script.onerror = script.onload = onScriptComplete;
 /******/ 		function onScriptComplete() {

diff --git a/Phraseanet-production-client/dist/authenticate.min.js b/Phraseanet-production-client/dist/authenticate.min.js
@@ -96,7 +96,7 @@ return /******/ (function(modules) { // webpackBootstrap
 /******/ 		if (__webpack_require__.nc) {
 /******/ 			script.setAttribute("nonce", __webpack_require__.nc);
 /******/ 		}
-/******/ 		script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".min.js?v=95";
+/******/ 		script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".min.js?v=96";
 /******/ 		var timeout = setTimeout(onScriptComplete, 120000);
 /******/ 		script.onerror = script.onload = onScriptComplete;
 /******/ 		function onScriptComplete() {

diff --git a/Phraseanet-production-client/dist/commons.js b/Phraseanet-production-client/dist/commons.js
@@ -91,7 +91,7 @@
 /******/ 		if (__webpack_require__.nc) {
 /******/ 			script.setAttribute("nonce", __webpack_require__.nc);
 /******/ 		}
-/******/ 		script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".js?v=95";
+/******/ 		script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".js?v=96";
 /******/ 		var timeout = setTimeout(onScriptComplete, 120000);
 /******/ 		script.onerror = script.onload = onScriptComplete;
 /******/ 		function onScriptComplete() {

diff --git a/Phraseanet-production-client/dist/commons.min.js b/Phraseanet-production-client/dist/commons.min.js
@@ -91,7 +91,7 @@
 /******/ 		if (__webpack_require__.nc) {
 /******/ 			script.setAttribute("nonce", __webpack_require__.nc);
 /******/ 		}
-/******/ 		script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".min.js?v=95";
+/******/ 		script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".min.js?v=96";
 /******/ 		var timeout = setTimeout(onScriptComplete, 120000);
 /******/ 		script.onerror = script.onload = onScriptComplete;
 /******/ 		function onScriptComplete() {

diff --git a/Phraseanet-production-client/dist/production.js b/Phraseanet-production-client/dist/production.js
@@ -7715,10 +7715,10 @@ var exportRecord = function exportRecord(services) {
             return false;
         });
 
-        (0, _jquery2.default)('input[name="obj[]"]', (0, _jquery2.default)('#download, #sendmail, #ftp')).bind('change', function () {
+        (0, _jquery2.default)('input.caption', (0, _jquery2.default)('#download, #sendmail, #ftp')).bind('change', function () {
             var $form = (0, _jquery2.default)(this).closest('form');
 
-            if ((0, _jquery2.default)('input.caption[name="obj[]"]:checked', $form).length > 0) {
+            if ((0, _jquery2.default)('input.caption:checked', $form).length > 0) {
                 (0, _jquery2.default)('div.businessfields', $form).show();
             } else {
                 (0, _jquery2.default)('div.businessfields', $form).hide();
@@ -7819,7 +7819,10 @@ var exportRecord = function exportRecord(services) {
         return true;
     }
 
-    return { initialize: initialize, openModal: openModal };
+    return {
+        initialize: initialize,
+        openModal: openModal
+    };
 };
 
 exports.default = exportRecord;

diff --git a/Phraseanet-production-client/dist/production.min.js b/Phraseanet-production-client/dist/production.min.js
@@ -7715,10 +7715,10 @@ var exportRecord = function exportRecord(services) {
             return false;
         });
 
-        (0, _jquery2.default)('input[name="obj[]"]', (0, _jquery2.default)('#download, #sendmail, #ftp')).bind('change', function () {
+        (0, _jquery2.default)('input.caption', (0, _jquery2.default)('#download, #sendmail, #ftp')).bind('change', function () {
             var $form = (0, _jquery2.default)(this).closest('form');
 
-            if ((0, _jquery2.default)('input.caption[name="obj[]"]:checked', $form).length > 0) {
+            if ((0, _jquery2.default)('input.caption:checked', $form).length > 0) {
                 (0, _jquery2.default)('div.businessfields', $form).show();
             } else {
                 (0, _jquery2.default)('div.businessfields', $form).hide();
@@ -7819,7 +7819,10 @@ var exportRecord = function exportRecord(services) {
         return true;
     }
 
-    return { initialize: initialize, openModal: openModal };
+    return {
+        initialize: initialize,
+        openModal: openModal
+    };
 };
 
 exports.default = exportRecord;

diff --git a/Phraseanet-production-client/package-lock.json b/Phraseanet-production-client/package-lock.json
diff --git a/Phraseanet-production-client/package.json b/Phraseanet-production-client/package.json
@@ -155,6 +155,7 @@
     "mapbox-gl-circle": "^1.6.5",
     "mapbox.js": "^2.4.0",
     "nouislider": "^9.2.0",
+    "pusher-js": "^8.3.0",
     "pym.js": "^1.3.1",
     "rx": "^4.1.0",
     "sprintf-js": "^1.1.1",