diff --git a/.env b/.env index 243d5d9137..529bbcce4e 100644 --- a/.env +++ b/.env @@ -64,6 +64,7 @@ # - "deleteRecord" # - "editRecord" # - "exportMail" +# - "downloadAsync" # - "exposeUpload" # - "exportFtp" # - "mainQueue" @@ -105,7 +106,7 @@ # Example with all profiles: # - COMPOSE_FILE=docker-compose.yml:docker-compose.datastores.yml:docker-compose.tools.yml # - COMPOSE_PROFILES=app,setup,gateway-classic,db,elasticsearch,redis,redis-session,rabbitmq,pma,mailhog,assetsInjest,createRecord,deleteRecord,editRecord, -# exportMail,exposeUpload,exportFtp,mainQueue,populateIndex,pullAssets,recordsActions,subdefCreation, +# exportMail,downloadAsync,exposeUpload,exportFtp,mainQueue,populateIndex,pullAssets,recordsActions,subdefCreation, # validationReminder,webhook,writeMetadatas,shareBasket,scheduler,elk,db-backup,phraseanet-saml-sp # @@ -197,7 +198,10 @@ GATEWAY_FASTCGI_HTTPS=off # Content Security Policy (CSP) # security that helps to detect and mitigate certain types of attacks, including Cross-Site Scripting ## @run -GATEWAY_CSP="default-src 'self' 127.0.0.1 https://apiws.carrick-skills.com:8443 https://apiws.carrick-flow.com:8443 https://fonts.gstatic.com *.tiles.mapbox.com https://api.mapbox.com https://events.mapbox.com *.axept.io *.matomo.cloud *.newrelic.com *.nr-data.net https://www.googletagmanager.com *.google-analytics.com *.phrasea.io https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 https://maxcdn.bootstrapcdn.com data: ; script-src 'unsafe-inline' 'unsafe-eval' 'self' https://www.gstatic.com *.alchemyasp.com *.axept.io *.matomo.cloud *.newrelic.com https://www.googletagmanager.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 https://maxcdn.bootstrapcdn.com data: blob: ; style-src 'self' 'unsafe-inline' https://fonts.gstatic.com https://fonts.googleapis.com https://www.google.com https://www.gstatic.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 https://maxcdn.bootstrapcdn.com ; img-src 'self' data: blob: *.tiles.mapbox.com https://axeptio.imgix.net *.cloudfront.net *.phrasea.io *.amazonaws.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 https://maxcdn.bootstrapcdn.com https://www.gnu.org/graphics/ ; object-src 'self'; frame-ancestors 'self'" +## GATEWAY_CSP="default-src 'self' 127.0.0.1 https://apiws.carrick-skills.com:8443 https://apiws.carrick-flow.com:8443 https://fonts.gstatic.com *.tiles.mapbox.com https://api.mapbox.com https://events.mapbox.com *.axept.io *.matomo.cloud *.newrelic.com *.nr-data.net https://www.googletagmanager.com *.google-analytics.com *.phrasea.io https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 https://maxcdn.bootstrapcdn.com data: ; script-src 'unsafe-inline' 'unsafe-eval' 'self' https://www.gstatic.com *.alchemyasp.com *.axept.io *.matomo.cloud *.newrelic.com https://www.googletagmanager.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 https://maxcdn.bootstrapcdn.com data: blob: ; style-src 'self' 'unsafe-inline' https://fonts.gstatic.com https://fonts.googleapis.com https://www.google.com https://www.gstatic.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 https://maxcdn.bootstrapcdn.com ; img-src 'self' data: blob: *.tiles.mapbox.com https://axeptio.imgix.net *.cloudfront.net *.phrasea.io *.amazonaws.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 https://maxcdn.bootstrapcdn.com https://www.gnu.org/graphics/ https://sockjs-eu.pusher.com:443 wss://ws-eu.pusher.com ; object-src 'self'; frame-ancestors 'self'" + + +GATEWAY_CSP="default-src 'self' 127.0.0.1 https://sockjs-eu.pusher.com:443 wss://ws-eu.pusher.com https://apiws.carrick-skills.com:8443 https://apiws.carrick-flow.com:8443 https://fonts.gstatic.com *.tiles.mapbox.com https://api.mapbox.com https://events.mapbox.com *.axept.io *.matomo.cloud *.newrelic.com *.nr-data.net https://www.googletagmanager.com *.google-analytics.com *.phrasea.io https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 data: ;script-src 'unsafe-inline' 'unsafe-eval' 'self' https://www.gstatic.com *.alchemyasp.com *.axept.io *.matomo.cloud *.newrelic.com https://www.googletagmanager.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 ;style-src 'self' 'unsafe-inline' https://fonts.gstatic.com https://fonts.googleapis.com https://www.google.com https://www.gstatic.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443;img-src 'self' data: blob: *.tiles.mapbox.com https://axeptio.imgix.net *.cloudfront.net *.phrasea.io *.amazonaws.com https://apiws.carrick-flow.com:8443 https://apiws.carrick-skills.com:8443 ; object-src 'self';frame-ancestors 'self'" # --- RabbitMQ settings ------------------------------------------------------------------------------------------------ @@ -414,6 +418,23 @@ DB_BACKUP_CRON_TIME= DB_BACKUP_GZIP_LEVEL=9 +# --- Pusher settings -------------------------------------------------------------------------------------- + +# Pusher settings used when PHRASEANET_DOWNLOAD_ASYNC=true (configuration.yml: download_async / enabled=true) + +# key +# @run +PUSHER_AUTH_KEY + +# secret +# @run +PUSHER_SECRET + +# app_id +# @run +PUSHER_APP_ID + + # --- Application cache settings --------------------------------------------------------------------------------------------------- # Cache setting type can be "redis" or "arraycache" @@ -463,6 +484,9 @@ PHRASEANET_ADMIN_ACCOUNT_EMAIL=admin@alchemy.fr # @run PHRASEANET_ADMIN_ACCOUNT_PASSWORD=iJRqXU0MwbyJewQLBbra6IWHsWly +# Use Pusher to enable async download. +# @run +PHRASEANET_DOWNLOAD_ASYNC=false # --- Phraseanet MySQL settings ---------------------------------------------------------------------------------------- @@ -686,6 +710,9 @@ PHRASEANET_WORKER_editRecord=2 # @run PHRASEANET_WORKER_exportMail=2 +# @run +PHRASEANET_WORKER_downloadAsync=2 + # @run PHRASEANET_WORKER_exposeUpload=2 diff --git a/Phraseanet-production-client/config/config.js b/Phraseanet-production-client/config/config.js index a80d4c2d74..4d14f1e9e7 100644 --- a/Phraseanet-production-client/config/config.js +++ b/Phraseanet-production-client/config/config.js @@ -13,5 +13,5 @@ module.exports = { setupDir: _root + 'tests/setup/node.js', karmaConf: _root + 'config/karma.conf.js', // change this version when you change JS file for lazy loading - assetFileVersion: 95 + assetFileVersion: 96 }; diff --git a/Phraseanet-production-client/dist/authenticate.js b/Phraseanet-production-client/dist/authenticate.js index c481923e4c..d0cca781fb 100644 --- a/Phraseanet-production-client/dist/authenticate.js +++ b/Phraseanet-production-client/dist/authenticate.js @@ -96,7 +96,7 @@ return /******/ (function(modules) { // webpackBootstrap /******/ if (__webpack_require__.nc) { /******/ script.setAttribute("nonce", __webpack_require__.nc); /******/ } -/******/ script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".js?v=95"; +/******/ script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".js?v=96"; /******/ var timeout = setTimeout(onScriptComplete, 120000); /******/ script.onerror = script.onload = onScriptComplete; /******/ function onScriptComplete() { diff --git a/Phraseanet-production-client/dist/authenticate.min.js b/Phraseanet-production-client/dist/authenticate.min.js index 8e97739776..52711bcba4 100644 --- a/Phraseanet-production-client/dist/authenticate.min.js +++ b/Phraseanet-production-client/dist/authenticate.min.js @@ -96,7 +96,7 @@ return /******/ (function(modules) { // webpackBootstrap /******/ if (__webpack_require__.nc) { /******/ script.setAttribute("nonce", __webpack_require__.nc); /******/ } -/******/ script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".min.js?v=95"; +/******/ script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".min.js?v=96"; /******/ var timeout = setTimeout(onScriptComplete, 120000); /******/ script.onerror = script.onload = onScriptComplete; /******/ function onScriptComplete() { diff --git a/Phraseanet-production-client/dist/commons.js b/Phraseanet-production-client/dist/commons.js index fcaf2f0225..4cc1a64f11 100644 --- a/Phraseanet-production-client/dist/commons.js +++ b/Phraseanet-production-client/dist/commons.js @@ -91,7 +91,7 @@ /******/ if (__webpack_require__.nc) { /******/ script.setAttribute("nonce", __webpack_require__.nc); /******/ } -/******/ script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".js?v=95"; +/******/ script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".js?v=96"; /******/ var timeout = setTimeout(onScriptComplete, 120000); /******/ script.onerror = script.onload = onScriptComplete; /******/ function onScriptComplete() { diff --git a/Phraseanet-production-client/dist/commons.min.js b/Phraseanet-production-client/dist/commons.min.js index ab39527929..75d2c8e4f3 100644 --- a/Phraseanet-production-client/dist/commons.min.js +++ b/Phraseanet-production-client/dist/commons.min.js @@ -91,7 +91,7 @@ /******/ if (__webpack_require__.nc) { /******/ script.setAttribute("nonce", __webpack_require__.nc); /******/ } -/******/ script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".min.js?v=95"; +/******/ script.src = __webpack_require__.p + "lazy-" + ({}[chunkId]||chunkId) + ".min.js?v=96"; /******/ var timeout = setTimeout(onScriptComplete, 120000); /******/ script.onerror = script.onload = onScriptComplete; /******/ function onScriptComplete() { diff --git a/Phraseanet-production-client/dist/production.js b/Phraseanet-production-client/dist/production.js index 1f4247784a..f1d762def3 100644 --- a/Phraseanet-production-client/dist/production.js +++ b/Phraseanet-production-client/dist/production.js @@ -7715,10 +7715,10 @@ var exportRecord = function exportRecord(services) { return false; }); - (0, _jquery2.default)('input[name="obj[]"]', (0, _jquery2.default)('#download, #sendmail, #ftp')).bind('change', function () { + (0, _jquery2.default)('input.caption', (0, _jquery2.default)('#download, #sendmail, #ftp')).bind('change', function () { var $form = (0, _jquery2.default)(this).closest('form'); - if ((0, _jquery2.default)('input.caption[name="obj[]"]:checked', $form).length > 0) { + if ((0, _jquery2.default)('input.caption:checked', $form).length > 0) { (0, _jquery2.default)('div.businessfields', $form).show(); } else { (0, _jquery2.default)('div.businessfields', $form).hide(); @@ -7819,7 +7819,10 @@ var exportRecord = function exportRecord(services) { return true; } - return { initialize: initialize, openModal: openModal }; + return { + initialize: initialize, + openModal: openModal + }; }; exports.default = exportRecord; diff --git a/Phraseanet-production-client/dist/production.min.js b/Phraseanet-production-client/dist/production.min.js index 1f4247784a..f1d762def3 100644 --- a/Phraseanet-production-client/dist/production.min.js +++ b/Phraseanet-production-client/dist/production.min.js @@ -7715,10 +7715,10 @@ var exportRecord = function exportRecord(services) { return false; }); - (0, _jquery2.default)('input[name="obj[]"]', (0, _jquery2.default)('#download, #sendmail, #ftp')).bind('change', function () { + (0, _jquery2.default)('input.caption', (0, _jquery2.default)('#download, #sendmail, #ftp')).bind('change', function () { var $form = (0, _jquery2.default)(this).closest('form'); - if ((0, _jquery2.default)('input.caption[name="obj[]"]:checked', $form).length > 0) { + if ((0, _jquery2.default)('input.caption:checked', $form).length > 0) { (0, _jquery2.default)('div.businessfields', $form).show(); } else { (0, _jquery2.default)('div.businessfields', $form).hide(); @@ -7819,7 +7819,10 @@ var exportRecord = function exportRecord(services) { return true; } - return { initialize: initialize, openModal: openModal }; + return { + initialize: initialize, + openModal: openModal + }; }; exports.default = exportRecord; diff --git a/Phraseanet-production-client/package-lock.json b/Phraseanet-production-client/package-lock.json index baed536794..7c4102f221 100644 --- a/Phraseanet-production-client/package-lock.json +++ b/Phraseanet-production-client/package-lock.json @@ -15005,6 +15005,21 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, + "pusher-js": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/pusher-js/-/pusher-js-8.3.0.tgz", + "integrity": "sha512-6GohP06WlVeomAQQe9qWh1IDzd3+InluWt+ZUOcecVK1SEQkg6a8uYVsvxSJm7cbccfmHhE0jDkmhKIhue8vmA==", + "requires": { + "tweetnacl": "^1.0.3" + }, + "dependencies": { + "tweetnacl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-1.0.3.tgz", + "integrity": "sha512-6rt+RN7aOi1nGMyC4Xa5DdYiukl2UWCbcJft7YhxReBGQD7OAM8Pbxw6YMo4r2diNEA8FEmu32YOn9rhaiE5yw==" + } + } + }, "pym.js": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/pym.js/-/pym.js-1.3.2.tgz", diff --git a/Phraseanet-production-client/package.json b/Phraseanet-production-client/package.json index b5c84e18d3..4b0909e594 100644 --- a/Phraseanet-production-client/package.json +++ b/Phraseanet-production-client/package.json @@ -155,6 +155,7 @@ "mapbox-gl-circle": "^1.6.5", "mapbox.js": "^2.4.0", "nouislider": "^9.2.0", + "pusher-js": "^8.3.0", "pym.js": "^1.3.1", "rx": "^4.1.0", "sprintf-js": "^1.1.1", diff --git a/Phraseanet-production-client/src/components/record/export.js b/Phraseanet-production-client/src/components/record/export.js index ada1d1fe66..324d33e0df 100644 --- a/Phraseanet-production-client/src/components/record/export.js +++ b/Phraseanet-production-client/src/components/record/export.js @@ -1,18 +1,23 @@ import $ from 'jquery'; import dialog from './../../phraseanet-common/components/dialog'; + const humane = require('humane-js'); const exportRecord = services => { - const { configService, localeService, appEvents } = services; - const url = configService.get('baseUrl'); - let $container = null; + const { + configService, + localeService, + appEvents + } = services; + const url = configService.get('baseUrl'); + let $container = null; const initialize = () => { $container = $('body'); $container.on('click', '.record-export-action', function (event) { event.preventDefault(); - let $el = $(event.currentTarget); - let key = ''; - let kind = $el.data('kind'); + let $el = $(event.currentTarget); + let key = ''; + let kind = $el.data('kind'); let idContent = $el.data('id'); switch (kind) { @@ -33,29 +38,30 @@ const exportRecord = services => { function doExport(datas) { var $dialog = dialog.create(services, { - size: 'Medium', + size: 'Medium', title: localeService.t('export') }); $.ajax({ - method: 'POST', - url: `${url}prod/export/multi-export/`, - data: datas, + method: 'POST', + url: `${url}prod/export/multi-export/`, + data: datas, success: function (data) { $dialog.setContent(data); if (window.exportConfig.isGuest) { dialog.get(1).close(); let guestModal = dialog.create( { - size: '500x100', + size: '500x100', closeOnEscape: true, - closeButton: false, - title: window.exportConfig.msg.modalTile + closeButton: false, + title: window.exportConfig.msg.modalTile }, 2 ); guestModal.setContent(window.exportConfig.msg.modalContent); - } else { + } + else { _onExportReady($dialog, window.exportConfig); } } @@ -83,7 +89,8 @@ const exportRecord = services => { .bind('change', function () { if ($(this).prop('checked')) { $(this).next().prop('disabled', false); - } else { + } + else { $(this).next().prop('disabled', true); } }); @@ -93,11 +100,11 @@ const exportRecord = services => { $('a.TOUview').bind('click', function (event) { event.preventDefault(); - let $el = $(event.currentTarget); + let $el = $(event.currentTarget); var options = { - size: 'Medium', + size: 'Medium', closeButton: true, - title: dataConfig.msg.termOfUseTitle + title: dataConfig.msg.termOfUseTitle }; let termOfuseDialog = dialog.create(services, options, 2); @@ -190,14 +197,15 @@ const exportRecord = services => { if (!data.error) { title = dataConfig.msg.success; - } else { + } + else { title = dataConfig.msg.warning; } var options = { - size: 'Alert', + size: 'Alert', closeButton: true, - title: title + title: title }; dialog.create(services, options, 2).setContent(data.msg); @@ -205,7 +213,8 @@ const exportRecord = services => { if (!data.error) { humane.info(data.msg); $dialog.close(); - } else { + } + else { humane.error(data.msg); } @@ -244,14 +253,15 @@ const exportRecord = services => { if (data.success) { humane.info(data.message); $dialog.close(); - } else { + } + else { var alert = dialog.create( services, { - size: 'Alert', + size: 'Alert', closeOnEscape: true, - closeButton: true, - title: dataConfig.msg.warning + closeButton: true, + title: dataConfig.msg.warning }, 2 ); @@ -278,11 +288,11 @@ const exportRecord = services => { $('#ftp .tryftp_button_loader').css('visibility', 'hidden'); var options = { - size: 'Alert', + size: 'Alert', closeButton: true, - title: data.success - ? dataConfig.msg.success - : dataConfig.msg.warning + title: data.success + ? dataConfig.msg.success + : dataConfig.msg.warning }; dialog @@ -297,7 +307,7 @@ const exportRecord = services => { }); $('#sendmail .sendmail_button').bind('click', function () { - if(!validEmail($('input[name="taglistdestmail"]', $('#sendmail')).val(), dataConfig)) { + if (!validEmail($('input[name="taglistdestmail"]', $('#sendmail')).val(), dataConfig)) { return false; } @@ -320,32 +330,30 @@ const exportRecord = services => { $dialog.close(); }); - $('.datepicker', $dialog.getDomElement()).datepicker({ - changeYear: true, - changeMonth: true, - dateFormat: 'yy-mm-dd' - }); + $('.datepicker', $dialog.getDomElement()) + .datepicker({ + changeYear: true, + changeMonth: true, + dateFormat: 'yy-mm-dd' + }); - $( - 'a.undisposable_link', - $dialog.getDomElement() - ).bind('click', function () { - $(this).parent().parent().find('.undisposable').slideToggle(); - return false; - }); + $('a.undisposable_link', $dialog.getDomElement()) + .bind('click', function () { + $(this).parent().parent().find('.undisposable').slideToggle(); + return false; + }); - $( - 'input[name="obj[]"]', - $('#download, #sendmail, #ftp') - ).bind('change', function () { - var $form = $(this).closest('form'); + $('input.caption', $('#download, #sendmail, #ftp')) + .bind('change', function () { + var $form = $(this).closest('form'); - if ($('input.caption[name="obj[]"]:checked', $form).length > 0) { - $('div.businessfields', $form).show(); - } else { - $('div.businessfields', $form).hide(); - } - }); + if ($('input.caption:checked', $form).length > 0) { + $('div.businessfields', $form).show(); + } + else { + $('div.businessfields', $form).hide(); + } + }); }; function validateEmail(email) { @@ -357,16 +365,16 @@ const exportRecord = services => { //split emailList by ; , or whitespace and filter empty element let emails = emailList.split(/[ ,;]+/).filter(Boolean); let alert; - for(let i=0; i < emails.length; i++) { + for (let i = 0; i < emails.length; i++) { if (!validateEmail(emails[i])) { alert = dialog.create( services, { - size: 'Alert', + size: 'Alert', closeOnEscape: true, - closeButton: true, - title: dataConfig.msg.warning + closeButton: true, + title: dataConfig.msg.warning }, 2 ); @@ -381,16 +389,16 @@ const exportRecord = services => { function check_TOU(container, dataConfig) { let checkbox = $('input[name="TOU_accept"]', $(container)); - let go = checkbox.length === 0 || checkbox.prop('checked'); + let go = checkbox.length === 0 || checkbox.prop('checked'); let alert; if (!go) { alert = dialog.create( services, { - size: 'Small', + size: 'Small', closeOnEscape: true, - closeButton: true, - title: dataConfig.msg.warning + closeButton: true, + title: dataConfig.msg.warning }, 2 ); @@ -403,7 +411,7 @@ const exportRecord = services => { } function check_subdefs(container, dataConfig) { - let go = false; + let go = false; let required = false; let alert; @@ -417,7 +425,8 @@ const exportRecord = services => { if ($.trim($(n).val()) === '') { required = true; $(n).addClass('error'); - } else { + } + else { $(n).removeClass('error'); } }); @@ -426,10 +435,10 @@ const exportRecord = services => { alert = dialog.create( services, { - size: 'Alert', + size: 'Alert', closeOnEscape: true, - closeButton: true, - title: dataConfig.msg.warning + closeButton: true, + title: dataConfig.msg.warning }, 2 ); @@ -442,10 +451,10 @@ const exportRecord = services => { alert = dialog.create( services, { - size: 'Alert', + size: 'Alert', closeOnEscape: true, - closeButton: true, - title: dataConfig.msg.warning + closeButton: true, + title: dataConfig.msg.warning }, 2 ); @@ -458,7 +467,10 @@ const exportRecord = services => { return true; } - return { initialize, openModal }; + return { + initialize, + openModal + }; }; export default exportRecord; diff --git a/composer.json b/composer.json index d1f80937f5..cbfd54d9b0 100644 --- a/composer.json +++ b/composer.json @@ -89,14 +89,14 @@ "hoa/dispatcher": "~0.0", "hoa/router": "~2.0", "igorw/get-in": "~1.0", - "imagine/imagine": "^0.10.0", + "imagine/imagine": "^0.11.0", "jms/serializer": "~0.10", "jms/translation-bundle": "dev-fix-2021-04-19", "justinrainbow/json-schema": "2.0.3 as 1.6.1", "league/flysystem": "^1.0", "league/flysystem-aws-s3-v2": "^1.0", "league/fractal": "dev-webgalleries#af1acc0275438571bc8c1d08a05a4b5af92c9f97 as 0.13.0", - "media-alchemyst/media-alchemyst": "^4.1.8", + "media-alchemyst/media-alchemyst": "^4.1.9", "monolog/monolog": "~1.3", "mrclay/minify": "~2.1.6", "neutron/process-manager": "2.0.x-dev@dev", @@ -133,7 +133,9 @@ "paragonie/random-lib": "^2.0", "czproject/git-php": "^3.17", "php-amqplib/php-amqplib": "^2.9", - "guzzlehttp/guzzle": " 6.3.3" + "guzzlehttp/guzzle": " 6.3.3", + "pusher/pusher-php-server": "^3.4", + "phpoffice/phpspreadsheet": "~1.8.0" }, "require-dev": { "mikey179/vfsstream": "~1.5", @@ -145,7 +147,9 @@ "": "lib/classes" } }, - "include-path": ["vendor/zend/gdata/library"], + "include-path": [ + "vendor/zend/gdata/library" + ], "extra": { "branch-alias": { "dev-master": "4.1.x-dev" diff --git a/composer.lock b/composer.lock index f15e7d1962..3e813cdb76 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cab652a03aad457cf7239d3156a0514c", + "content-hash": "9e7490439544a7184da9a64bd9f3012f", "packages": [ { "name": "alchemy-fr/tcpdf-clone", @@ -3706,16 +3706,16 @@ }, { "name": "imagine/imagine", - "version": "v0.10.0", + "version": "v0.11.0", "source": { "type": "git", "url": "https://github.com/alchemy-fr/Imagine.git", - "reference": "352a4bbf72c34f9a4b8990da790d65d82ff9542a" + "reference": "7ac6a20e9b267f1835a2a5e4a5075bb4a47b4d90" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/alchemy-fr/Imagine/zipball/352a4bbf72c34f9a4b8990da790d65d82ff9542a", - "reference": "352a4bbf72c34f9a4b8990da790d65d82ff9542a", + "url": "https://api.github.com/repos/alchemy-fr/Imagine/zipball/7ac6a20e9b267f1835a2a5e4a5075bb4a47b4d90", + "reference": "7ac6a20e9b267f1835a2a5e4a5075bb4a47b4d90", "shasum": "" }, "require": { @@ -3759,9 +3759,9 @@ "image processing" ], "support": { - "source": "https://github.com/alchemy-fr/Imagine/tree/v0.10.0" + "source": "https://github.com/alchemy-fr/Imagine/tree/v0.11.0" }, - "time": "2023-09-14T14:43:24+00:00" + "time": "2023-10-02T17:20:35+00:00" }, { "name": "ircmaxell/password-compat", @@ -4341,24 +4341,196 @@ ], "time": "2016-12-02T14:55:48+00:00" }, + { + "name": "markbaker/complex", + "version": "1.5.0", + "source": { + "type": "git", + "url": "https://github.com/MarkBaker/PHPComplex.git", + "reference": "c3131244e29c08d44fefb49e0dd35021e9e39dd2" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/MarkBaker/PHPComplex/zipball/c3131244e29c08d44fefb49e0dd35021e9e39dd2", + "reference": "c3131244e29c08d44fefb49e0dd35021e9e39dd2", + "shasum": "" + }, + "require": { + "php": "^5.6.0|^7.0" + }, + "require-dev": { + "dealerdirect/phpcodesniffer-composer-installer": "^0.5.0", + "phpcompatibility/php-compatibility": "^9.0", + "phpdocumentor/phpdocumentor": "2.*", + "phploc/phploc": "^4.0|^5.0|^6.0|^7.0", + "phpmd/phpmd": "2.*", + "phpunit/phpunit": "^4.8.35|^5.0|^6.0|^7.0", + "sebastian/phpcpd": "2.*", + "squizlabs/php_codesniffer": "^3.4.0" + }, + "type": "library", + "autoload": { + "files": [ + "classes/src/functions/abs.php", + "classes/src/functions/acos.php", + "classes/src/functions/acosh.php", + "classes/src/functions/acot.php", + "classes/src/functions/acoth.php", + "classes/src/functions/acsc.php", + "classes/src/functions/acsch.php", + "classes/src/functions/argument.php", + "classes/src/functions/asec.php", + "classes/src/functions/asech.php", + "classes/src/functions/asin.php", + "classes/src/functions/asinh.php", + "classes/src/functions/atan.php", + "classes/src/functions/atanh.php", + "classes/src/functions/conjugate.php", + "classes/src/functions/cos.php", + "classes/src/functions/cosh.php", + "classes/src/functions/cot.php", + "classes/src/functions/coth.php", + "classes/src/functions/csc.php", + "classes/src/functions/csch.php", + "classes/src/functions/exp.php", + "classes/src/functions/inverse.php", + "classes/src/functions/ln.php", + "classes/src/functions/log2.php", + "classes/src/functions/log10.php", + "classes/src/functions/negative.php", + "classes/src/functions/pow.php", + "classes/src/functions/rho.php", + "classes/src/functions/sec.php", + "classes/src/functions/sech.php", + "classes/src/functions/sin.php", + "classes/src/functions/sinh.php", + "classes/src/functions/sqrt.php", + "classes/src/functions/tan.php", + "classes/src/functions/tanh.php", + "classes/src/functions/theta.php", + "classes/src/operations/add.php", + "classes/src/operations/subtract.php", + "classes/src/operations/multiply.php", + "classes/src/operations/divideby.php", + "classes/src/operations/divideinto.php" + ], + "psr-4": { + "Complex\\": "classes/src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Mark Baker", + "email": "mark@lange.demon.co.uk" + } + ], + "description": "PHP Class for working with complex numbers", + "homepage": "https://github.com/MarkBaker/PHPComplex", + "keywords": [ + "complex", + "mathematics" + ], + "support": { + "issues": "https://github.com/MarkBaker/PHPComplex/issues", + "source": "https://github.com/MarkBaker/PHPComplex/tree/1.5.0" + }, + "time": "2020-08-26T19:47:57+00:00" + }, + { + "name": "markbaker/matrix", + "version": "1.2.3", + "source": { + "type": "git", + "url": "https://github.com/MarkBaker/PHPMatrix.git", + "reference": "44bb1ab01811116f01fe216ab37d921dccc6c10d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/MarkBaker/PHPMatrix/zipball/44bb1ab01811116f01fe216ab37d921dccc6c10d", + "reference": "44bb1ab01811116f01fe216ab37d921dccc6c10d", + "shasum": "" + }, + "require": { + "php": "^5.6.0|^7.0.0" + }, + "require-dev": { + "dealerdirect/phpcodesniffer-composer-installer": "dev-master", + "phpcompatibility/php-compatibility": "dev-master", + "phploc/phploc": "^4", + "phpmd/phpmd": "dev-master", + "phpunit/phpunit": "^5.7|^6.0|7.0", + "sebastian/phpcpd": "^3.0", + "squizlabs/php_codesniffer": "^3.0@dev" + }, + "type": "library", + "autoload": { + "files": [ + "classes/src/Functions/adjoint.php", + "classes/src/Functions/antidiagonal.php", + "classes/src/Functions/cofactors.php", + "classes/src/Functions/determinant.php", + "classes/src/Functions/diagonal.php", + "classes/src/Functions/identity.php", + "classes/src/Functions/inverse.php", + "classes/src/Functions/minors.php", + "classes/src/Functions/trace.php", + "classes/src/Functions/transpose.php", + "classes/src/Operations/add.php", + "classes/src/Operations/directsum.php", + "classes/src/Operations/subtract.php", + "classes/src/Operations/multiply.php", + "classes/src/Operations/divideby.php", + "classes/src/Operations/divideinto.php" + ], + "psr-4": { + "Matrix\\": "classes/src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Mark Baker", + "email": "mark@lange.demon.co.uk" + } + ], + "description": "PHP Class for working with matrices", + "homepage": "https://github.com/MarkBaker/PHPMatrix", + "keywords": [ + "mathematics", + "matrix", + "vector" + ], + "support": { + "issues": "https://github.com/MarkBaker/PHPMatrix/issues", + "source": "https://github.com/MarkBaker/PHPMatrix/tree/1.2.3" + }, + "time": "2021-01-26T14:36:01+00:00" + }, { "name": "media-alchemyst/media-alchemyst", - "version": "v4.1.8", + "version": "v4.1.9", "source": { "type": "git", "url": "https://github.com/alchemy-fr/Media-Alchemyst.git", - "reference": "9097b7b074afc28aa4d5a83e7a2e59609c59b617" + "reference": "aa55169c838f30b8f76210ca6970e34444d1e716" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/alchemy-fr/Media-Alchemyst/zipball/9097b7b074afc28aa4d5a83e7a2e59609c59b617", - "reference": "9097b7b074afc28aa4d5a83e7a2e59609c59b617", + "url": "https://api.github.com/repos/alchemy-fr/Media-Alchemyst/zipball/aa55169c838f30b8f76210ca6970e34444d1e716", + "reference": "aa55169c838f30b8f76210ca6970e34444d1e716", "shasum": "" }, "require": { "alchemy/ghostscript": "~0.4.0", "alchemy/mediavorus": "^0.4.4", - "imagine/imagine": "^0.10.0", + "imagine/imagine": "^0.11.0", "monolog/monolog": "~1.0", "neutron/temporary-filesystem": "^2.1.1", "php": ">=5.3.3", @@ -4415,9 +4587,9 @@ ], "support": { "issues": "https://github.com/alchemy-fr/Media-Alchemyst/issues", - "source": "https://github.com/alchemy-fr/Media-Alchemyst/tree/v4.1.8" + "source": "https://github.com/alchemy-fr/Media-Alchemyst/tree/v4.1.9" }, - "time": "2023-09-14T15:36:28+00:00" + "time": "2023-10-12T16:33:49+00:00" }, { "name": "monolog/monolog", @@ -5479,6 +5651,104 @@ ], "time": "2015-05-17T12:39:23+00:00" }, + { + "name": "phpoffice/phpspreadsheet", + "version": "1.8.2", + "source": { + "type": "git", + "url": "https://github.com/PHPOffice/PhpSpreadsheet.git", + "reference": "0c1346a1956347590b7db09533966307d20cb7cc" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/PHPOffice/PhpSpreadsheet/zipball/0c1346a1956347590b7db09533966307d20cb7cc", + "reference": "0c1346a1956347590b7db09533966307d20cb7cc", + "shasum": "" + }, + "require": { + "ext-ctype": "*", + "ext-dom": "*", + "ext-fileinfo": "*", + "ext-gd": "*", + "ext-iconv": "*", + "ext-libxml": "*", + "ext-mbstring": "*", + "ext-simplexml": "*", + "ext-xml": "*", + "ext-xmlreader": "*", + "ext-xmlwriter": "*", + "ext-zip": "*", + "ext-zlib": "*", + "markbaker/complex": "^1.4", + "markbaker/matrix": "^1.1", + "php": "^5.6|^7.0", + "psr/simple-cache": "^1.0" + }, + "require-dev": { + "doctrine/instantiator": "^1.0.0", + "dompdf/dompdf": "^0.8.0", + "friendsofphp/php-cs-fixer": "@stable", + "jpgraph/jpgraph": "^4.0", + "mpdf/mpdf": "^7.0.0", + "phpcompatibility/php-compatibility": "^8.0", + "phpunit/phpunit": "^5.7", + "squizlabs/php_codesniffer": "^3.3", + "tecnickcom/tcpdf": "^6.2" + }, + "suggest": { + "dompdf/dompdf": "Option for rendering PDF with PDF Writer", + "jpgraph/jpgraph": "Option for rendering charts, or including charts with PDF or HTML Writers", + "mpdf/mpdf": "Option for rendering PDF with PDF Writer", + "tecnickcom/tcpdf": "Option for rendering PDF with PDF Writer" + }, + "type": "library", + "autoload": { + "psr-4": { + "PhpOffice\\PhpSpreadsheet\\": "src/PhpSpreadsheet" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "LGPL-2.1-or-later" + ], + "authors": [ + { + "name": "Erik Tilt" + }, + { + "name": "Adrien Crivelli" + }, + { + "name": "Maarten Balliauw", + "homepage": "https://blog.maartenballiauw.be" + }, + { + "name": "Mark Baker", + "homepage": "https://markbakeruk.net" + }, + { + "name": "Franck Lefevre", + "homepage": "https://rootslabs.net" + } + ], + "description": "PHPSpreadsheet - Read, Create and Write Spreadsheet documents in PHP - Spreadsheet engine", + "homepage": "https://github.com/PHPOffice/PhpSpreadsheet", + "keywords": [ + "OpenXML", + "excel", + "gnumeric", + "ods", + "php", + "spreadsheet", + "xls", + "xlsx" + ], + "support": { + "issues": "https://github.com/PHPOffice/PhpSpreadsheet/issues", + "source": "https://github.com/PHPOffice/PhpSpreadsheet/tree/1.8.2" + }, + "time": "2019-07-08T21:21:25+00:00" + }, { "name": "phpoption/phpoption", "version": "1.5.0", @@ -5810,6 +6080,115 @@ ], "time": "2016-10-10T12:19:37+00:00" }, + { + "name": "psr/simple-cache", + "version": "1.0.1", + "source": { + "type": "git", + "url": "https://github.com/php-fig/simple-cache.git", + "reference": "408d5eafb83c57f6365a3ca330ff23aa4a5fa39b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/simple-cache/zipball/408d5eafb83c57f6365a3ca330ff23aa4a5fa39b", + "reference": "408d5eafb83c57f6365a3ca330ff23aa4a5fa39b", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\SimpleCache\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common interfaces for simple caching", + "keywords": [ + "cache", + "caching", + "psr", + "psr-16", + "simple-cache" + ], + "support": { + "source": "https://github.com/php-fig/simple-cache/tree/master" + }, + "time": "2017-10-23T01:57:42+00:00" + }, + { + "name": "pusher/pusher-php-server", + "version": "v3.4.1", + "source": { + "type": "git", + "url": "https://github.com/pusher/pusher-http-php.git", + "reference": "a5fcdc65efd8d9a8291efbe01d326ec7ef5d5cee" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/pusher/pusher-http-php/zipball/a5fcdc65efd8d9a8291efbe01d326ec7ef5d5cee", + "reference": "a5fcdc65efd8d9a8291efbe01d326ec7ef5d5cee", + "shasum": "" + }, + "require": { + "ext-curl": "*", + "paragonie/sodium_compat": "^1.6", + "php": ">=5.4 <7.4", + "psr/log": "^1.0" + }, + "require-dev": { + "phpunit/phpunit": "^4.8.35 || ^5.7" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.4-dev" + } + }, + "autoload": { + "psr-4": { + "Pusher\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "Library for interacting with the Pusher REST API", + "keywords": [ + "events", + "messaging", + "php-pusher-server", + "publish", + "push", + "pusher", + "real time", + "real-time", + "realtime", + "rest", + "trigger" + ], + "support": { + "issues": "https://github.com/pusher/pusher-http-php/issues", + "source": "https://github.com/pusher/pusher-http-php/tree/master" + }, + "time": "2019-03-19T11:19:11+00:00" + }, { "name": "ramsey/uuid", "version": "3.5.2", diff --git a/config/configuration.sample.yml b/config/configuration.sample.yml index afb0abf8f1..9173e61f5b 100644 --- a/config/configuration.sample.yml +++ b/config/configuration.sample.yml @@ -244,6 +244,12 @@ registration-fields: - name: geonameid required: true +download_async: + enabled: true +pusher: + auth_key: 'pusher-auth_key' + secret: 'pusher-secret' + app_id: 'pusher-app_id' xsendfile: enabled: false type: nginx diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 629c4cb8f7..3c7cbe351a 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -99,8 +99,15 @@ services: w-exportMail: volumes: - - ../:/var/alchemy - - .:/var/alchemy/Phraseanet + - ../:/var/alchemy + - .:/var/alchemy/Phraseanet + networks: + - internal + + w-downloadAsync: + volumes: + - ../:/var/alchemy + - .:/var/alchemy/Phraseanet networks: - internal diff --git a/docker-compose.yml b/docker-compose.yml index 991fb0c4bb..1d031a38e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -309,6 +309,7 @@ services: - PHRASEANET_WORKER_deleteRecord - PHRASEANET_WORKER_editRecord - PHRASEANET_WORKER_exportMail + - PHRASEANET_WORKER_downloadAsync - PHRASEANET_WORKER_exposeUpload - PHRASEANET_WORKER_ftp - PHRASEANET_WORKER_mainQueue @@ -629,38 +630,92 @@ services: profiles: ["workers", "exportMail"] restart: on-failure depends_on: - - phraseanet + - phraseanet environment: - - STACK_NAME - - OPCACHE_ENABLED - - SESSION_CACHE_LIMITER - - PHP_LOG_LEVEL - - PHP_CLI_MEMORY_LIMIT - - LC_MESSAGES=C.UTF-8 - - LC_COLLATE=C.UTF-8 - - LC_IDENTIFICATION=C.UTF-8 - - LANG=C.UTF-8 - - LC_MEASUREMENT=C.UTF-8 - - LC_CTYPE=C.UTF-8 - - LC_TIME=C.UTF-8 - - LC_NAME=C.UTF-8 - - PHRASEANET_EXPLODE_WORKER - - PHRASEANET_WORKERS_LAUNCH_METHOD - - PHRASEANET_WORKER_exportMail - - IMAGEMAGICK_POLICY_VERSION - - IMAGEMAGICK_POLICY_WIDTH - - IMAGEMAGICK_POLICY_HEIGHT - - IMAGEMAGICK_POLICY_MAP - - IMAGEMAGICK_POLICY_MEMORY - - IMAGEMAGICK_POLICY_AREA - - IMAGEMAGICK_POLICY_DISK - - IMAGEMAGICK_POLICY_TEMPORARY_PATH - - NEWRELIC_ENABLED - - NEWRELIC_LICENSE_KEY - - NEWRELIC_APP_NAME - - BLACKFIRE_ENABLED - - BLACKFIRE_SERVER_ID - - BLACKFIRE_SERVER_TOKEN + - STACK_NAME + - OPCACHE_ENABLED + - SESSION_CACHE_LIMITER + - PHP_LOG_LEVEL + - PHP_CLI_MEMORY_LIMIT + - LC_MESSAGES=C.UTF-8 + - LC_COLLATE=C.UTF-8 + - LC_IDENTIFICATION=C.UTF-8 + - LANG=C.UTF-8 + - LC_MEASUREMENT=C.UTF-8 + - LC_CTYPE=C.UTF-8 + - LC_TIME=C.UTF-8 + - LC_NAME=C.UTF-8 + - PHRASEANET_EXPLODE_WORKER + - PHRASEANET_WORKERS_LAUNCH_METHOD + - PHRASEANET_WORKER_exportMail + - IMAGEMAGICK_POLICY_VERSION + - IMAGEMAGICK_POLICY_WIDTH + - IMAGEMAGICK_POLICY_HEIGHT + - IMAGEMAGICK_POLICY_MAP + - IMAGEMAGICK_POLICY_MEMORY + - IMAGEMAGICK_POLICY_AREA + - IMAGEMAGICK_POLICY_DISK + - IMAGEMAGICK_POLICY_TEMPORARY_PATH + - NEWRELIC_ENABLED + - NEWRELIC_LICENSE_KEY + - NEWRELIC_APP_NAME + - BLACKFIRE_ENABLED + - BLACKFIRE_SERVER_ID + - BLACKFIRE_SERVER_TOKEN + volumes: + - ${PHRASEANET_CONFIG_DIR}:/var/alchemy/Phraseanet/config:rw + - ${PHRASEANET_LOGS_DIR}:/var/alchemy/Phraseanet/logs:rw + - ${PHRASEANET_DATA_DIR}:/var/alchemy/Phraseanet/datas:rw + - ${PHRASEANET_THUMBNAILS_DIR}:/var/alchemy/Phraseanet/www/thumbnails:rw + - ${PHRASEANET_CUSTOM_DIR}:/var/alchemy/Phraseanet/www/custom:rw + - ${PHRASEANET_CACHE_DIR}:/var/alchemy/Phraseanet/cache:rw + - ${PHRASEANET_TMP_DIR}:/var/alchemy/Phraseanet/tmp:rw + networks: + - internal + + w-downloadAsync: + build: + context: . + target: phraseanet-worker + args: + - SSH_PRIVATE_KEY=${PHRASEANET_SSH_PRIVATE_KEY} + - PHRASEANET_PLUGINS=${PHRASEANET_PLUGINS} + image: $PHRASEANET_DOCKER_REGISTRY/phraseanet-worker:$PHRASEANET_DOCKER_TAG + profiles: ["workers", "downloadAsync"] + restart: on-failure + depends_on: + - phraseanet + environment: + - STACK_NAME + - OPCACHE_ENABLED + - SESSION_CACHE_LIMITER + - PHP_LOG_LEVEL + - PHP_CLI_MEMORY_LIMIT + - LC_MESSAGES=C.UTF-8 + - LC_COLLATE=C.UTF-8 + - LC_IDENTIFICATION=C.UTF-8 + - LANG=C.UTF-8 + - LC_MEASUREMENT=C.UTF-8 + - LC_CTYPE=C.UTF-8 + - LC_TIME=C.UTF-8 + - LC_NAME=C.UTF-8 + - PHRASEANET_EXPLODE_WORKER + - PHRASEANET_WORKERS_LAUNCH_METHOD + - PHRASEANET_WORKER_downloadAsync + - IMAGEMAGICK_POLICY_VERSION + - IMAGEMAGICK_POLICY_WIDTH + - IMAGEMAGICK_POLICY_HEIGHT + - IMAGEMAGICK_POLICY_MAP + - IMAGEMAGICK_POLICY_MEMORY + - IMAGEMAGICK_POLICY_AREA + - IMAGEMAGICK_POLICY_DISK + - IMAGEMAGICK_POLICY_TEMPORARY_PATH + - NEWRELIC_ENABLED + - NEWRELIC_LICENSE_KEY + - NEWRELIC_APP_NAME + - BLACKFIRE_ENABLED + - BLACKFIRE_SERVER_ID + - BLACKFIRE_SERVER_TOKEN volumes: - ${PHRASEANET_CONFIG_DIR}:/var/alchemy/Phraseanet/config:rw - ${PHRASEANET_LOGS_DIR}:/var/alchemy/Phraseanet/logs:rw diff --git a/docker/phraseanet/setup/entrypoint.sh b/docker/phraseanet/setup/entrypoint.sh index 11bbeb982c..5b70408f70 100755 --- a/docker/phraseanet/setup/entrypoint.sh +++ b/docker/phraseanet/setup/entrypoint.sh @@ -177,6 +177,11 @@ if [[ -f "$FILE" && $PHRASEANET_SETUP = 1 ]]; then bin/setup system:config set -q workers.queue.worker-queue.user $PHRASEANET_RABBITMQ_USER bin/setup system:config set -q workers.queue.worker-queue.password $PHRASEANET_RABBITMQ_PASSWORD + echo `date +"%Y-%m-%d %H:%M:%S"` " - Phraseanet setting DOWNLOAD_ASYNC & PUSHER" + bin/setup system:config set download_async.enabled $PHRASEANET_DOWNLOAD_ASYNC + bin/setup system:config set pusher.auth_key $PUSHER_AUTH_KEY + bin/setup system:config set pusher.secret $PUSHER_SECRET + bin/setup system:config set pusher.app_id $PUSHER_APP_ID diff --git a/docker/phraseanet/worker/supervisor_conf.d/downloadAsync.conf b/docker/phraseanet/worker/supervisor_conf.d/downloadAsync.conf new file mode 100644 index 0000000000..f412ca9493 --- /dev/null +++ b/docker/phraseanet/worker/supervisor_conf.d/downloadAsync.conf @@ -0,0 +1,28 @@ +[program:w-downloadAsync] +command=nice -n 15 /usr/local/bin/php /var/alchemy/Phraseanet/bin/console worker:execute --queue-name=downloadAsync -m $PHRASEANET_WORKER_downloadAsync ; the program (relative uses PATH, can take args) +stdout_logfile=AUTO ; stdout log path, NONE for none; default AUTO +stderr_logfile=AUTO ; stderr log path, NONE for none; default AUTO +process_name=%(program_name)s ; process_name expr (default %(program_name)s) +numprocs=1 ; number of processes copies to start (def 1) +directory=/tmp ; directory to cwd to before exec (def no cwd) +priority=999 ; the relative start priority (default 999) +autostart=true ; start at supervisord start (default: true) +autorestart=true ; whether/when to restart (default: unexpected) +startsecs=0 ; number of secs prog must stay running (def. 1) +startretries=3 ; max # of serial start failures (default 3) +exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) +stopsignal=INT ; signal used to kill process (default TERM) +stopwaitsecs=20 ; max num secs to wait b4 SIGKILL (default 10) +stopasgroup=true ; send stop signal to the UNIX process group (default false) +killasgroup=true ; SIGKILL the UNIX process group (def false) +redirect_stderr=true ; redirect proc stderr to stdout (default false) +user=1000 ; setuid to this UNIX account to run the program +stdout_logfile_maxbytes=50MB ; max # logfile bytes b4 rotation (default 50MB) +stdout_logfile_backups=10 ; # of stdout logfile backups (default 10) +stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) +stdout_events_enabled=false ; emit events on stdout writes (default false) +stderr_logfile_maxbytes=10MB ; max # logfile bytes b4 rotation (default 50MB) +stderr_logfile_backups=10 ; # of stderr logfile backups (default 10) +stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) +stderr_events_enabled=false ; emit events on stderr writes (default false) +environment=HOME=/home/app,USER=app ; process environment additions (def no adds) diff --git a/lib/Alchemy/Phrasea/Controller/Prod/DoDownloadController.php b/lib/Alchemy/Phrasea/Controller/Prod/DoDownloadController.php index 66a14d57fc..659b522a31 100644 --- a/lib/Alchemy/Phrasea/Controller/Prod/DoDownloadController.php +++ b/lib/Alchemy/Phrasea/Controller/Prod/DoDownloadController.php @@ -71,6 +71,7 @@ public function prepareDownload(Request $request, Token $token) } return new Response($this->render( + /** @uses templates/web/prod/actions/Download/prepare.html.twig */ '/prod/actions/Download/prepare.html.twig', [ 'module_name' => $this->app->trans('Export'), 'module' => $this->app->trans('Export'), @@ -106,7 +107,7 @@ public function downloadDocuments(Token $token) $exportName = $list['export_name']; - if ($list['count'] === 1) { + if ($list['count'] === 1 && !$list['cgu']) { $file = end($list['files']); $subdef = end($file['subdefs']); $exportName = sprintf('%s%s.%s', $file['export_name'], $subdef['ajout'], $subdef['exportExt']); diff --git a/lib/Alchemy/Phrasea/Controller/Prod/DownloadController.php b/lib/Alchemy/Phrasea/Controller/Prod/DownloadController.php index c575c86431..55d08f9a2e 100644 --- a/lib/Alchemy/Phrasea/Controller/Prod/DownloadController.php +++ b/lib/Alchemy/Phrasea/Controller/Prod/DownloadController.php @@ -11,11 +11,16 @@ use Alchemy\Phrasea\Application\Helper\DispatcherAware; use Alchemy\Phrasea\Controller\Controller; +use Alchemy\Phrasea\Core\Configuration\PropertyAccess; +use Alchemy\Phrasea\Core\Event\DownloadAsyncEvent; use Alchemy\Phrasea\Core\Event\ExportEvent; use Alchemy\Phrasea\Core\PhraseaEvents; use Alchemy\Phrasea\Model\Manipulator\TokenManipulator; +use set_export; +use Symfony\Component\HttpFoundation\JsonResponse; use Symfony\Component\HttpFoundation\RedirectResponse; use Symfony\Component\HttpFoundation\Request; +use Symfony\Component\HttpFoundation\Response; class DownloadController extends Controller { @@ -54,13 +59,136 @@ public function checkDownload(Request $request) $list['export_name'] = sprintf('%s.zip', $download->getExportName()); $token = $this->getTokenManipulator()->createDownloadToken($this->getAuthenticatedUser(), serialize($list)); + $this->getDispatcher()->dispatch(PhraseaEvents::EXPORT_CREATE, new ExportEvent( - $this->getAuthenticatedUser(), $ssttid, $lst, $subdefs, $download->getExportName()) + $this->getAuthenticatedUser(), + $ssttid, + $lst, + $subdefs, + $download->getExportName() + ) ); + /** @see DoDownloadController::prepareDownload */ return $this->app->redirectPath('prepare_download', ['token' => $token->getValue()]); } + /** + * display the downloasAsync page + * + * @param Request $request + * @return Response + */ + public function listDownloadAsync(Request $request) + { + if (!$this->isCrsfValid($request, 'prodExportDownload')) { + $this->app->abort(403); + } + + $lst = $request->request->get('lst'); + $ssttid = $request->request->get('ssttid', ''); + $subdefs = $request->request->get('obj', []); + + $download = new \set_export($this->app, $lst, $ssttid); + + if (0 === $download->get_total_download()) { + $this->app->abort(403); + } + + // "stamp_choice" is a ckbox with value "NO_STAMP" to "remove stamp" on download + $stamp_method = set_export::STAMP_ASYNC; // will not stamp, but flag files to be stamped + if($request->request->get('stamp_choice') === set_export::NO_STAMP) { + $stamp_method = set_export::NO_STAMP; + } + + $list = $download->prepare_export( + $this->getAuthenticatedUser(), + $this->app['filesystem'], + $subdefs, + $request->request->get('type') === 'title' ? true : false, + $request->request->get('businessfields'), + // do not stamp now, worker will do + $stamp_method, + true + ); + $list['export_name'] = sprintf('%s.zip', $download->getExportName()); + $list['include_report'] = $request->request->get('include_report') === 'INCLUDE_REPORT'; + $list['include_businessfields'] = (bool)$request->request->get('businessfields'); + + $records = []; + + foreach ($list['files'] as $file) { + if (!is_array($file) || !isset($file['base_id']) || !isset($file['record_id'])) { + continue; + } + $sbasId = \phrasea::sbasFromBas($this->app, $file['base_id']); + + try { + $record = new \record_adapter($this->app, $sbasId, $file['record_id']); + } catch (\Exception $e) { + continue; + } + + $records[sprintf('%s_%s', $sbasId, $file['record_id'])] = $record; + } + + $token = $this->getTokenManipulator()->createDownloadToken($this->getAuthenticatedUser(), serialize($list)); + + $pusher_auth_key =$this->getConf()->get(['download_async', 'enabled'], false) ? $this->getConf()->get(['pusher', 'auth_key'], '') : null; + return new Response($this->render( + /** @uses templates/web/prod/actions/Download/prepare_async.html.twig */ + '/prod/actions/Download/prepare_async.html.twig', [ + 'module_name' => $this->app->trans('Export'), + 'module' => $this->app->trans('Export'), + 'list' => $list, + 'records' => $records, + 'token' => $token, + 'anonymous' => $request->query->get('anonymous', false), + 'type' => $request->query->get('type', \Session_Logger::EVENT_EXPORTDOWNLOAD), + 'pusher_auth_key' => $pusher_auth_key, + 'csrfToken' => $this->getSession()->get('prodExportDownload_token'), + ])); + } + + + /** + * @param Request $request + * @return JsonResponse|void + * @throws \Doctrine\ORM\NonUniqueResultException + */ + public function startDownloadAsync(Request $request) + { + if (!$this->isCrsfValid($request, 'prodExportDownload')) { + $this->app->abort(403); + } + + try { + $token = $this->getTokenManipulator()->findValidToken($request->request->get('token', "")); + + if ($token) { + // ask the worker to build the zip + $this->dispatch(PhraseaEvents::DOWNLOAD_ASYNC_CREATE, new DownloadAsyncEvent( + $token->getUser()->getId(), + $token->getValue(), + [ + ] + )); + + return new JsonResponse([ + 'success' => true, + 'token' => $token->getValue() + ]); + } + else { + throw new \Exception("invalid or expired token"); + } + } + catch(\Exception $e) { + // no-op + $this->app->abort(403, $e->getMessage()); + } + } + /** * @return TokenManipulator */ @@ -68,4 +196,20 @@ private function getTokenManipulator() { return $this->app['manipulator.token']; } + + /** + * @return PropertyAccess + */ + protected function getConf() + { + return $this->app['conf']; + } + + /** + * @return PropertyAccess + */ + protected function getSession() + { + return $this->app['session']; + } } diff --git a/lib/Alchemy/Phrasea/ControllerProvider/Prod/DoDownload.php b/lib/Alchemy/Phrasea/ControllerProvider/Prod/DoDownload.php index e776e53152..2459930dfc 100644 --- a/lib/Alchemy/Phrasea/ControllerProvider/Prod/DoDownload.php +++ b/lib/Alchemy/Phrasea/ControllerProvider/Prod/DoDownload.php @@ -46,16 +46,19 @@ public function connect(Application $app) { $controllers = $this->createCollection($app); + /** @uses DoDownloadController::prepareDownload */ $controllers->get('/{token}/prepare/', 'controller.prod.do-download:prepareDownload') ->before($app['middleware.token.converter']) ->bind('prepare_download') ->assert('token', '[a-zA-Z0-9]{8,32}'); + /** @uses DoDownloadController::downloadDocuments */ $controllers->match('/{token}/get/', 'controller.prod.do-download:downloadDocuments') ->before($app['middleware.token.converter']) ->bind('document_download') ->assert('token', '[a-zA-Z0-9]{8,32}'); + /** @uses DoDownloadController::downloadExecute */ $controllers->post('/{token}/execute/', 'controller.prod.do-download:downloadExecute') ->before($app['middleware.token.converter']) ->bind('execute_download') diff --git a/lib/Alchemy/Phrasea/ControllerProvider/Prod/Download.php b/lib/Alchemy/Phrasea/ControllerProvider/Prod/Download.php index 04df8b9010..0c30192a9a 100644 --- a/lib/Alchemy/Phrasea/ControllerProvider/Prod/Download.php +++ b/lib/Alchemy/Phrasea/ControllerProvider/Prod/Download.php @@ -45,6 +45,15 @@ public function connect(Application $app) $controllers->before(new OAuthListener(['exit_not_present' => false])); $this->getFirewall($app)->addMandatoryAuthentication($controllers); + /** @uses DownloadController::listDownloadAsync */ + $controllers->post('/list_async/', 'controller.prod.download:listDownloadAsync') + ->bind('list_download_async'); + + /** @uses DownloadController::startDownloadAsync */ + $controllers->post('/start_async/', 'controller.prod.download:startDownloadAsync') + ->bind('start_download_async'); + + /** @uses DownloadController::checkDownload */ $controllers->post('/', 'controller.prod.download:checkDownload') ->bind('check_download'); diff --git a/lib/Alchemy/Phrasea/ControllerProvider/Prod/Export.php b/lib/Alchemy/Phrasea/ControllerProvider/Prod/Export.php index d830c12daf..b6326e6270 100644 --- a/lib/Alchemy/Phrasea/ControllerProvider/Prod/Export.php +++ b/lib/Alchemy/Phrasea/ControllerProvider/Prod/Export.php @@ -49,15 +49,19 @@ public function connect(Application $app) $controllers->before(new OAuthListener(['exit_not_present' => false])); $this->getFirewall($app)->addMandatoryAuthentication($controllers); + /** @uses ExportController::displayMultiExport */ $controllers->post('/multi-export/', 'controller.prod.export:displayMultiExport') ->bind('export_multi_export'); + /** @uses ExportController::exportMail */ $controllers->post('/mail/', 'controller.prod.export:exportMail') ->bind('export_mail'); + /** @uses ExportController::exportFtp */ $controllers->post('/ftp/', 'controller.prod.export:exportFtp') ->bind('export_ftp'); + /** @uses ExportController::testFtpConnexion */ $controllers->post('/ftp/test/', 'controller.prod.export:testFtpConnexion') ->bind('export_ftp_test'); diff --git a/lib/Alchemy/Phrasea/Core/Event/DownloadAsyncEvent.php b/lib/Alchemy/Phrasea/Core/Event/DownloadAsyncEvent.php new file mode 100644 index 0000000000..55bce794dc --- /dev/null +++ b/lib/Alchemy/Phrasea/Core/Event/DownloadAsyncEvent.php @@ -0,0 +1,48 @@ +userId = $userId; + $this->tokenValue = $tokenValue; + $this->params = $params; + } + + public function getTokenValue() + { + return $this->tokenValue; + } + + public function getParams() + { + return $this->params; + } + + /** + * @return mixed + */ + public function getUserId() + { + return $this->userId; + } +} diff --git a/lib/Alchemy/Phrasea/Core/PhraseaEvents.php b/lib/Alchemy/Phrasea/Core/PhraseaEvents.php index 7f231d3802..8f9d31d833 100644 --- a/lib/Alchemy/Phrasea/Core/PhraseaEvents.php +++ b/lib/Alchemy/Phrasea/Core/PhraseaEvents.php @@ -52,6 +52,7 @@ final class PhraseaEvents const EXPORT_MAIL_FAILURE = 'export.mail-failure'; const EXPORT_CREATE = 'export.create'; const EXPORT_MAIL_CREATE = 'export.mail-create'; + const DOWNLOAD_ASYNC_CREATE = 'download.async-create'; const RECORD_EDIT = 'record.edit'; const RECORD_UPLOAD = 'record.upload'; diff --git a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php index 9a22276c23..91e4b4d2c9 100644 --- a/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php +++ b/lib/Alchemy/Phrasea/Model/Manipulator/TokenManipulator.php @@ -19,6 +19,7 @@ use Alchemy\Phrasea\Model\Repositories\TokenRepository; use DateTime; use Doctrine\Common\Persistence\ObjectManager; +use Doctrine\ORM\NonUniqueResultException; use RandomLib\Generator; use RuntimeException; @@ -96,6 +97,17 @@ public function create($user, $type, $expiration = null, $data = null) return $token; } + /** + * @param string $tokenValue + * @return Token + * @throws NonUniqueResultException + */ + public function findValidToken(string $tokenValue) + { + return $this->repository->findValidToken($tokenValue); + } + + /** * @param Basket $basket * @param User $user diff --git a/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php b/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php index ffca44fd5d..a2a2391d84 100644 --- a/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php +++ b/lib/Alchemy/Phrasea/Twig/PhraseanetExtension.php @@ -62,7 +62,7 @@ public function getGlobals() { return [ // change this version when you change JS file to force the navigation to reload js file - 'assetFileVersion' => 95 + 'assetFileVersion' => 96 ]; } diff --git a/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php b/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php index 76011a76a3..b5c8df60d6 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Provider/AlchemyWorkerServiceProvider.php @@ -9,6 +9,7 @@ use Alchemy\Phrasea\WorkerManager\Worker\AssetsIngestWorker; use Alchemy\Phrasea\WorkerManager\Worker\CreateRecordWorker; use Alchemy\Phrasea\WorkerManager\Worker\DeleteRecordWorker; +use Alchemy\Phrasea\WorkerManager\Worker\DownloadAsyncWorker; use Alchemy\Phrasea\WorkerManager\Worker\EditRecordWorker; use Alchemy\Phrasea\WorkerManager\Worker\ExportMailWorker; use Alchemy\Phrasea\WorkerManager\Worker\ExposeUploadWorker; @@ -108,6 +109,12 @@ public function register(Application $app) ->setDelivererLocator(new LazyLocator($app, 'notification.deliverer')); })); + $app['alchemy_worker.type_based_worker_resolver']->addFactory(MessagePublisher::DOWNLOAD_ASYNC_TYPE, new CallableWorkerFactory(function () use ($app) { + return (new DownloadAsyncWorker($app, $app['conf'])) + ->setFileSystemLocator(new LazyLocator($app, 'filesystem')) + ->setDelivererLocator(new LazyLocator($app, 'notification.deliverer')); + })); + $app['alchemy_worker.type_based_worker_resolver']->addFactory(MessagePublisher::ASSETS_INGEST_TYPE, new CallableWorkerFactory(function () use ($app) { return (new AssetsIngestWorker($app)) ->setEntityManagerLocator(new LazyLocator($app, 'orm.em')); diff --git a/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php b/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php index cfad841912..43d6ff48a0 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Queue/AMQPConnection.php @@ -69,6 +69,11 @@ class AMQPConnection self::MAX_RETRY => self::DEFAULT_MAX_RETRY_VALUE, self::TTL_RETRY => self::DEFAULT_RETRY_DELAY_VALUE, ], + MessagePublisher::DOWNLOAD_ASYNC_TYPE => [ + 'with' => self::WITH_RETRY, + self::MAX_RETRY => self::DEFAULT_MAX_RETRY_VALUE, + self::TTL_RETRY => self::DEFAULT_RETRY_DELAY_VALUE, + ], MessagePublisher::EXPOSE_UPLOAD_TYPE => [ 'with' => self::WITH_RETRY, self::MAX_RETRY => self::DEFAULT_MAX_RETRY_VALUE, diff --git a/lib/Alchemy/Phrasea/WorkerManager/Queue/MessagePublisher.php b/lib/Alchemy/Phrasea/WorkerManager/Queue/MessagePublisher.php index 57c5049d2a..476bdadb5f 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Queue/MessagePublisher.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Queue/MessagePublisher.php @@ -15,6 +15,7 @@ class MessagePublisher const CREATE_RECORD_TYPE = 'createRecord'; const DELETE_RECORD_TYPE = 'deleteRecord'; const EXPORT_MAIL_TYPE = 'exportMail'; + const DOWNLOAD_ASYNC_TYPE = 'downloadAsync'; const EXPOSE_UPLOAD_TYPE = 'exposeUpload'; const FTP_TYPE = 'ftp'; const POPULATE_INDEX_TYPE = 'populateIndex'; diff --git a/lib/Alchemy/Phrasea/WorkerManager/Subscriber/ExportSubscriber.php b/lib/Alchemy/Phrasea/WorkerManager/Subscriber/ExportSubscriber.php index bbfde318bd..25ba2250d6 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Subscriber/ExportSubscriber.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Subscriber/ExportSubscriber.php @@ -2,6 +2,7 @@ namespace Alchemy\Phrasea\WorkerManager\Subscriber; +use Alchemy\Phrasea\Core\Event\DownloadAsyncEvent; use Alchemy\Phrasea\Core\Event\ExportMailEvent; use Alchemy\Phrasea\Core\PhraseaEvents; use Alchemy\Phrasea\WorkerManager\Event\ExportFtpEvent; @@ -20,6 +21,20 @@ public function __construct(MessagePublisher $messagePublisher) $this->messagePublisher = $messagePublisher; } + public function onDownloadAsyncCreate(DownloadAsyncEvent $event) + { + $payload = [ + 'message_type' => MessagePublisher::DOWNLOAD_ASYNC_TYPE, + 'payload' => [ + 'userId' => $event->getUserId(), + 'tokenValue' => $event->getTokenValue(), + 'params' => serialize($event->getParams()) + ] + ]; + + $this->messagePublisher->publishMessage($payload, MessagePublisher::DOWNLOAD_ASYNC_TYPE); + } + public function onExportMailCreate(ExportMailEvent $event) { $payload = [ @@ -73,6 +88,7 @@ public function onExportFtp(ExportFtpEvent $event) public static function getSubscribedEvents() { return [ + PhraseaEvents::DOWNLOAD_ASYNC_CREATE => 'onDownloadAsyncCreate', PhraseaEvents::EXPORT_MAIL_CREATE => 'onExportMailCreate', WorkerEvents::EXPORT_MAIL_FAILURE => 'onExportMailFailure', WorkerEvents::EXPORT_FTP => 'onExportFtp' diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/DownloadAsyncWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/DownloadAsyncWorker.php new file mode 100644 index 0000000000..15ff57b0e1 --- /dev/null +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/DownloadAsyncWorker.php @@ -0,0 +1,446 @@ +app = $app; + $this->conf = $conf; + } + + public function process(array $payload) + { + $this->repoWorkerJob = $this->getWorkerRunningJobRepository(); + $em = $this->repoWorkerJob->getEntityManager(); + $em->beginTransaction(); + $this->repoWorkerJob->reconnect(); + $date = new \DateTime(); + + $message = [ + 'message_type' => MessagePublisher::DOWNLOAD_ASYNC_TYPE, + 'payload' => $payload + ]; + + try { + $workerRunningJob = new WorkerRunningJob(); + $workerRunningJob + ->setWork(MessagePublisher::DOWNLOAD_ASYNC_TYPE) + ->setPayload($message) + ->setPublished($date->setTimestamp($payload['published'])) + ->setStatus(WorkerRunningJob::RUNNING) + ; + + $em->persist($workerRunningJob); + + $em->flush(); + + $em->commit(); + } catch (\Exception $e) { + $em->rollback(); + $workerRunningJob = null; + } + + $filesystem = $this->getFilesystem(); + + $params = unserialize($payload['params']); + + /** @var UserRepository $userRepository */ + $userRepository = $this->app['repo.users']; + + $user = $userRepository->find($payload['userId']); + $localeEmitter = $user->getLocale(); + + /** @var TokenRepository $tokenRepository */ + $tokenRepository = $this->app['repo.tokens']; + + /** @var Token $token */ + $token = $tokenRepository->findValidToken($payload['tokenValue']); + + if($this->conf->get(['download_async', 'enabled'], false)) { + $options = array( + 'cluster' => 'eu', + 'useTLS' => true + ); + try { + $this->pusher = new Pusher( + $this->conf->get(['pusher', 'auth_key'], ''), + $this->conf->get(['pusher', 'secret'], ''), + $this->conf->get(['pusher', 'app_id'], ''), + $options + ); + $this->pusher_channel_name = $token->getValue(); + } + catch (\Exception $e) { + // no-op + } + } + + $list = unserialize($token->getData()); + + $caption_dir = null; + $spreadsheet = null; + + if($list['include_report']) { + if (!$caption_dir) { + // do this only once + $caption_dir = $this->app['tmp.caption.path'] . '/' . time() . $payload['userId'] . '/'; + $filesystem->mkdir($caption_dir, 0750); + } + $spreadsheet = new Spreadsheet(); + } + + $totalSize = 0; + + $worksheet_ref_by_db = []; + + foreach($list['files'] as $k_file => $v_file) { + $record = null; + $databox_id = $v_file['databox_id']; + $record_id = $v_file['record_id']; + + if($spreadsheet) { + if(!$record) { + $record = $this->app->getApplicationBox()->get_databox($databox_id)->get_record($record_id); + } + if(!array_key_exists($databox_id, $worksheet_ref_by_db)) { + // Create a new worksheet with db name + $ws = new Worksheet($spreadsheet, $this->app->getApplicationBox()->get_databox($databox_id)->get_dbname()); + $spreadsheet->addSheet($ws); + if(count($worksheet_ref_by_db) === 0) { + // we just added the first ws, we can delete the "default" one + $spreadsheet->removeSheetByIndex(0); + } + + $include_businessfields = false; + if ($list['include_businessfields'] && $this->app->getAclForUser($user)->has_right_on_base($record->getBaseId(), \ACL::CANMODIFRECORD)) { + $include_businessfields = true; + } + + // add fields names as first row + $max_col = $col = 1; + + $ref = $this->cellRefFromColumnAndRow($col, 1); + $ws->setCellValue($ref, "[record_id]"); + $max_col = $col++; + + $ref = $this->cellRefFromColumnAndRow($col, 1); + $ws->setCellValue($ref, "[file]"); + $max_col = $col++; + + $field_columns = []; + foreach ($record->getDatabox()->get_meta_structure() as $field) { + if($include_businessfields || !$field->isBusiness()) { + $field_columns[$field->get_name()] = $col; + $ref = $this->cellRefFromColumnAndRow($col, 1); + $ws->setCellValue($ref, $field->get_name()); + $max_col = $col++; + } + } + // freeze the title row + $ws->freezePane("A2"); + + $worksheet_ref_by_db[$databox_id] = [ + 'worksheet_index' => $spreadsheet->getIndex($ws), + 'worksheet' => $ws, + 'row' => 2, + 'max_col' => $max_col, + 'max_row' => 1, + 'field_columns' => $field_columns, + ]; + } + + // add a row for the record + $ws_ref = &$worksheet_ref_by_db[$databox_id]; + /** @var Worksheet $ws */ + $ws = $ws_ref['worksheet']; + + $ref = $this->cellRefFromColumnAndRow(1, $ws_ref['row']); + $ws->setCellValue($ref, $record_id); + + $ref = $this->cellRefFromColumnAndRow(2, $ws_ref['row']); + $ws->setCellValue($ref, $v_file['export_name']); + + $max_lines = 0; + foreach ($record->get_caption()->get_fields([], $include_businessfields) as $field) { + if(array_key_exists($field->get_name(), $ws_ref['field_columns'])) { + $col = $ws_ref['field_columns'][$field->get_name()]; + $value = join($field->get_values(), "\n"); + $ref = $this->cellRefFromColumnAndRow($col, $ws_ref['row']); + $ws->setCellValue($ref, $value); + // empiric: max number of "lines" in this row + if(($n_lines = substr_count($value, "\n") + 1) > $max_lines) { + $max_lines = $n_lines; + } + } + } + // empiric: adjust the "height" of the row (@see https://phpspreadsheet.readthedocs.io/en/latest/topics/recipes/) + $h = 14.5 * min(100, $max_lines) ; + $ws->getRowDimension($ws_ref['row'])->setRowHeight($h); + + $ws_ref['max_row'] = $ws_ref['row']; + $ws_ref['row']++; + } + + foreach($v_file['subdefs'] as $k_subdef => $v_subdef) { + if($k_subdef === "document" && $v_subdef['to_stamp']) { + // we must stamp this document + try { + if(!$record) { + $record = $this->app->getApplicationBox()->get_databox($v_file['databox_id'])->get_record($v_file['record_id']); + } + $sd = $record->get_subdef($k_subdef); + if(!is_null($path = \recordutils_image::stamp($this->app, $sd))) { + // stamped ! + $pi = pathinfo($path); + $list['files'][$k_file]['subdefs'][$k_subdef]['path'] = $pi['dirname']; + $list['files'][$k_file]['subdefs'][$k_subdef]['file'] = $pi['basename']; + $list['files'][$k_file]['subdefs'][$k_subdef]['size'] = filesize($path); + } + } + catch (\Exception $e) { + // failed to stamp ? ignore and send the original file + } + } + if($list['files'][$k_file]['subdefs'][$k_subdef]['size'] > 0) { + $totalSize += $list['files'][$k_file]['subdefs'][$k_subdef]['size']; + $this->push( + 'file_ok', + [ + 'message' => "", + 'databox_id' => $list['files'][$k_file]['databox_id'], + 'record_id' => $list['files'][$k_file]['record_id'], + 'subdef' => $k_subdef, + 'size' => $list['files'][$k_file]['subdefs'][$k_subdef]['size'], + 'human_size' => $this->getHumanSize($list['files'][$k_file]['subdefs'][$k_subdef]['size']), + 'total_size' => $totalSize, + 'human_total_size' => $this->getHumanSize($totalSize), + ] + ); + } + } + } + + // add the captions files if exist + foreach ($list['captions'] as $v_caption) { + if (!$caption_dir) { + // do this only once + $caption_dir = $this->app['tmp.caption.path'] . '/' . time() . $payload['userId'] . '/'; + $filesystem->mkdir($caption_dir, 0750); + } + + $subdefName = $v_caption['subdefName']; + $kFile = $v_caption['fileId']; + + $download_element = new \record_exportElement( + $this->app, + $list['files'][$kFile]['databox_id'], + $list['files'][$kFile]['record_id'], + $v_caption['elementDirectory'], + $v_caption['remain_hd'], + $user + ); + + $file = $list['files'][$kFile]["export_name"] + . $list['files'][$kFile]["subdefs"][$subdefName]["ajout"] . '.' + . $list['files'][$kFile]["subdefs"][$subdefName]["exportExt"]; + + $desc = $this->app['serializer.caption']->serialize($download_element->get_caption(), $v_caption['serializeMethod'], $v_caption['businessFields']); + file_put_contents($caption_dir . $file, $desc); + + $list['files'][$kFile]["subdefs"][$subdefName]["path"] = $caption_dir; + $list['files'][$kFile]["subdefs"][$subdefName]["file"] = $file; + $list['files'][$kFile]["subdefs"][$subdefName]["size"] = filesize($caption_dir . $file); + $list['files'][$kFile]["subdefs"][$subdefName]['businessfields'] = $v_caption['businessFields']; + + $totalSize += $list['files'][$kFile]["subdefs"][$subdefName]["size"]; + $this->push( + 'file_ok', + [ + 'message' => "", + 'databox_id' => $list['files'][$kFile]['databox_id'], + 'record_id' => $list['files'][$kFile]['record_id'], + 'subdef' => $subdefName, + 'size' => $list['files'][$kFile]["subdefs"][$subdefName]["size"], + 'human_size' => $this->getHumanSize($list['files'][$kFile]["subdefs"][$subdefName]["size"]), + 'total_size' => $totalSize, + 'human_total_size' => $this->getHumanSize($totalSize), + ] + ); + } + + if($spreadsheet) { + + $style_title = [ + 'font' => [ + 'bold' => true, + ], + 'alignment' => [ + 'horizontal' => \PhpOffice\PhpSpreadsheet\Style\Alignment::HORIZONTAL_CENTER, + 'vertical' => \PhpOffice\PhpSpreadsheet\Style\Alignment::VERTICAL_TOP + ], + 'borders' => [ + 'bottom' => [ + 'borderStyle' => \PhpOffice\PhpSpreadsheet\Style\Border::BORDER_THIN, + ], + ], + 'fill' => [ + 'fillType' => \PhpOffice\PhpSpreadsheet\Style\Fill::FILL_SOLID, + 'color' => [ + 'argb' => 'FFA0A0A0', + ] + ], + ]; + $style_values = [ + 'alignment' => [ + 'vertical' => \PhpOffice\PhpSpreadsheet\Style\Alignment::VERTICAL_TOP + ], + ]; + + foreach($worksheet_ref_by_db as $databox_id => &$ws_ref) { + /** @var Worksheet $ws */ + $ws = $ws_ref['worksheet']; + $range = "A1:" . $this->cellRefFromColumnAndRow($ws_ref['max_col'], 1); + $ws->getStyle($range)->applyFromArray($style_title); + $range = "A2:" . $this->cellRefFromColumnAndRow($ws_ref['max_col'], $ws_ref['max_row']); + $ws->getStyle($range)->applyFromArray($style_values); + for($col=1; $col<=$ws_ref['max_col']; $col++) { + $range = $this->cellRefFromColumnAndRow($col); // no row in range = whole column (ex. "A") + $ws->getColumnDimension($range)->setAutoSize(true); + } + }; + + $file = 'report.xlsx'; + + $writer = new Xlsx($spreadsheet); + $writer->save($caption_dir . $file); + + unset($writer); + unset($spreadsheet); + $spreadsheet = null; + + $list['files']['report'] = [ + "export_name" => 'report', + 'subdefs' => [ + 'report' => [ + "ajout" => '', + "exportExt" => 'xlsx', + "label" => '', + "path" => $caption_dir, + "file" => $file, + "to_stamp" => false, + "size" => filesize($caption_dir . $file), + "mime" => '', + "folder" => '' + ] + ] + ]; + + $totalSize += $list['files']['report']["subdefs"]['report']["size"]; + } + + $this->repoWorkerJob->reconnect(); + //zip documents + \set_export::build_zip( + $this->app, + $token, + $list, + $this->app['tmp.download.path'].'/'. $token->getValue() . '.zip' + ); + + if ($workerRunningJob != null) { + $this->repoWorkerJob->reconnect(); + $workerRunningJob + ->setStatus(WorkerRunningJob::FINISHED) + ->setFinished(new \DateTime('now')) + ; + + $em->persist($workerRunningJob); + + $em->flush(); + } + + sleep(1); + + $this->push('zip_ready', ['message' => ""]); + + } + + private function push(string $event, $data) + { + if($this->pusher) { + $r = $this->pusher->trigger( + $this->pusher_channel_name, + $event, + $data + ); + } + } + + // todo : this Ko;Mo;Go code already exists in phraseanet (download) + private function getHumanSize(int $size) + { + $unit = 'octets'; + $units = ['Go', 'Mo', 'Ko']; + $format = "%d %s"; + while ($size > 1024 && !empty($units)) { + $unit = array_pop($units); + $size /= 1024.0; + $format = "%.02f %s"; + } + return sprintf($format, $size, $unit); + } + + + /** + * @return WorkerRunningJobRepository + */ + private function getWorkerRunningJobRepository() + { + return $this->app['repo.worker-running-job']; + } + + private function cellRefFromColumnAndRow(int $col, int $row = null) + { + $r = Coordinate::stringFromColumnIndex($col); + if($row !== null) { + $r .= $row; + } + + return $r; + } +} diff --git a/lib/Alchemy/Phrasea/WorkerManager/Worker/ExportMailWorker.php b/lib/Alchemy/Phrasea/WorkerManager/Worker/ExportMailWorker.php index f3faaad652..f60849dbfd 100644 --- a/lib/Alchemy/Phrasea/WorkerManager/Worker/ExportMailWorker.php +++ b/lib/Alchemy/Phrasea/WorkerManager/Worker/ExportMailWorker.php @@ -18,6 +18,7 @@ use Alchemy\Phrasea\WorkerManager\Event\ExportMailFailureEvent; use Alchemy\Phrasea\WorkerManager\Event\WorkerEvents; use Alchemy\Phrasea\WorkerManager\Queue\MessagePublisher; +use Pusher\Pusher; class ExportMailWorker implements WorkerInterface { @@ -227,6 +228,19 @@ public function process(array $payload) $em->flush(); } + sleep(30); + $options = array( + 'cluster' => 'eu', + 'useTLS' => true + ); + $pusher = new Pusher( + '07b97d8d50b1f2b3d515', + 'c441cc58dbf1f51f3e0c', + '1682224', + $options + ); + $data['message'] = 'hello world'; + $pusher->trigger('my-channel', 'my-event', $data); } /** diff --git a/lib/classes/set/export.php b/lib/classes/set/export.php index 9975ae7de1..99a34df139 100644 --- a/lib/classes/set/export.php +++ b/lib/classes/set/export.php @@ -421,6 +421,7 @@ public function prepare_export(User $user, Filesystem $filesystem, Array $wanted $file_names = []; $size = 0; $unicode = $this->app['unicode']; + $hasCgu = false; /** @var record_exportElement $download_element */ foreach ($this->elements as $download_element) { @@ -436,6 +437,10 @@ public function prepare_export(User $user, Filesystem $filesystem, Array $wanted 'subdefs' => [], ]; + if (!$hasCgu && !PDFCgu::isDataboxCguEmpty($this->app, $download_element->getDataboxId())) { + $hasCgu = true; + } + $BF = false; if ($includeBusinessFields && $this->app->getAclForUser($user)->has_right_on_base($download_element->getBaseId(), \ACL::CANMODIFRECORD)) { @@ -467,7 +472,7 @@ public function prepare_export(User $user, Filesystem $filesystem, Array $wanted // build the export_name // if ($rename_title) { - // use the title (may be a concat of fields) + // use the title (can be a concat of fields) $export_name = strip_tags($download_element->get_title(['removeExtension' => true, 'encode'=> record_adapter::ENCODE_FOR_URI])); // if the "title" ends up with a "filename-like" field, remove extension if (strtolower(substr($export_name, -strlen($extension)-1)) === '.'.strtolower($extension)) { @@ -694,7 +699,7 @@ public function prepare_export(User $user, Filesystem $filesystem, Array $wanted . $files[$id]["subdefs"][$subdefName]["ajout"] . '.' . $files[$id]["subdefs"][$subdefName]["exportExt"]; - $desc = $this->app['serializer.caption']->serialize($download_element->get_caption(), $serializeMethod, $BF); + $desc = $this->getCaptionSerializer()->serialize($download_element->get_caption(), $serializeMethod, $BF); file_put_contents($caption_dir . $file, $desc); $files[$id]["subdefs"][$subdefName]["path"] = $caption_dir; @@ -712,6 +717,7 @@ public function prepare_export(User $user, Filesystem $filesystem, Array $wanted 'names' => $file_names, 'size' => $size, 'count' => $n_files, + 'cgu' => $hasCgu, ]; return $this->list; @@ -745,7 +751,9 @@ public static function build_zip(Application $app, Token $token, array $list, $z // group recordId per databoxId foreach ($files as $file) { - $recordIdsPerDatabox[$file['databox_id']][] = $file['record_id']; + if(array_key_exists('databox_id', $file)) { + $recordIdsPerDatabox[$file['databox_id']][] = $file['record_id']; + } } foreach ($files as $record) { @@ -766,7 +774,7 @@ public static function build_zip(Application $app, Token $token, array $list, $z $toRemove[] = $path; } - if (!in_array($record['databox_id'], $databoxIds)) { + if (array_key_exists('databox_id', $record) && !in_array($record['databox_id'], $databoxIds)) { // add also the databox cgu in the zip $databoxIds[] = $record['databox_id']; @@ -827,6 +835,10 @@ public static function log_download(Application $app, array $list, $type, $anony ]) ? $type : Session_Logger::EVENT_EXPORTDOWNLOAD; foreach ($files as $record) { + if(!array_key_exists('base_id', $record)) { + // a "non-record" file, like xlsx report + continue; + } foreach ($record["subdefs"] as $o => $obj) { $sbas_id = phrasea::sbasFromBas($app, $record['base_id']); @@ -894,4 +906,12 @@ public function has_stamp_option() return false; } + + /** + * @return CaptionSerializer + */ + private function getCaptionSerializer() + { + return $this->app['serializer.caption']; + } } diff --git a/lib/conf.d/configuration.yml b/lib/conf.d/configuration.yml index dab5d447e5..9e2a635252 100644 --- a/lib/conf.d/configuration.yml +++ b/lib/conf.d/configuration.yml @@ -258,6 +258,12 @@ registration-fields: - name: geonameid required: true +download_async: + enabled: false +pusher: + auth_key: 'pusher-auth_key' + secret: 'pusher-secret' + app_id: 'pusher-app_id' xsendfile: enabled: false type: nginx diff --git a/package.json b/package.json index c9852eb936..ed013eae32 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "normalize-css": "^2.1.0", "npm": "^6.0.0", "npm-modernizr": "^2.8.3", + "pusher-js": "^8.3.0", "requirejs": "^2.3.5", "tinymce": "^4.0.28", "underscore": "^1.8.3", diff --git a/resources/gulp/build.js b/resources/gulp/build.js index 80ef15dcf6..9ac912d5e5 100644 --- a/resources/gulp/build.js +++ b/resources/gulp/build.js @@ -50,6 +50,7 @@ gulp.task('build-vendors', [ 'build-jquery-lazyload', 'build-jquery-test-paths', 'build-simple-colorpicker', - 'build-jquery-datetimepicker' + 'build-jquery-datetimepicker', + 'build-pusher-js' ], function () { }); diff --git a/resources/gulp/components/vendors/pusher-js.js b/resources/gulp/components/vendors/pusher-js.js new file mode 100644 index 0000000000..c22b3dbdba --- /dev/null +++ b/resources/gulp/components/vendors/pusher-js.js @@ -0,0 +1,8 @@ +var gulp = require('gulp'); +var config = require('../../config.js'); +var utils = require('../../utils.js'); + +gulp.task('build-pusher-js', [], function(){ + return gulp.src([config.paths.nodes + 'pusher-js/dist/web/**']) + .pipe(gulp.dest(config.paths.build + 'vendors/pusher-js')); +}); diff --git a/templates/web/common/dialog_export.html.twig b/templates/web/common/dialog_export.html.twig index b99a837708..f72d9601be 100644 --- a/templates/web/common/dialog_export.html.twig +++ b/templates/web/common/dialog_export.html.twig @@ -105,10 +105,18 @@ {% endif %} {% if download.get_total_download() > 0 %} +

{{ 'export:: telechargement' | trans }}

-
+ {% if app['conf'].get(['download_async', 'enabled'], false) %} + {# \Alchemy\Phrasea\Controller\Prod\DownloadController::checkDownloadAsync #} + {% set download_path = 'list_download_async' %} + {% else %} + {# \Alchemy\Phrasea\Controller\Prod\DownloadController::checkDownload #} + {% set download_path = 'check_download' %} + {% endif %} + {% for name, values in download.get_display_download() %} @@ -137,6 +145,12 @@
{% endif %} {% endfor %} +
+ +
{% if download.has_business_fields_access() %} {% endif %} {% if app['conf'].get(['registry', 'actions', 'export-stamp-choice']) == true and download.has_stamp_option() == true %} -
+
+

{{ 'export:: envoi par mail' | trans }}

+ {# \Alchemy\Phrasea\Controller\Prod\ExportController::exportMail #}
-
- {{ 'export:email:info:: email addresses separated by commas' | trans }} -
+
+ {{ 'export:email:info:: email addresses separated by commas' | trans }} +
{{ 'export::mail: destinataire' | trans }} @@ -189,10 +205,10 @@ {% set my_email = app.getAuthenticatedUser().getEmail() %}
{% if my_email != '' %} - + {% else %}