diff --git a/docs/apps/datafeeder.md b/docs/apps/datafeeder.md index 11a19844a4..521414345d 100644 --- a/docs/apps/datafeeder.md +++ b/docs/apps/datafeeder.md @@ -4,6 +4,29 @@ outline: deep # Datafeeder -## Chapter 1 +"Datafeeder" is a geOrchestra's backend RESTful service to upload file based datasets and publish them to GeoServer and GeoNetwork in one shot. -## Chapter 2 +The separate front-end UI service provides the wizard-like user interface to interact with this [backend](https://github.com/georchestra/georchestra/tree/master/datafeeder). + +The front-end part is built with Geonetwork-UI components + +## Usage + +The easiest way to deploy datafeeder locally is to use the docker composition in `support-services/datafeeder`. + +```shell +cd support-services/datafeeder +docker compose up -d +``` + +In `geonetwork-ui/` directory, run : + +```shell +npx nx run datafeeder:serve +``` + +## Configuration + +### Adding projections and encodings + +The datafeeder configuration can be edited by modifying the `frontend-config.json` file in `support-services/datafeeder/datadir/datafeeder/` folder. diff --git a/proxy-config.js b/proxy-config.js index e93aea2ec2..e492ffdbc8 100644 --- a/proxy-config.js +++ b/proxy-config.js @@ -18,12 +18,22 @@ module.exports = { router: (req) => decodeURIComponent(req._parsedUrl.query), }, '/datafeeder': { - target: 'http://localhost:8080', + target: 'http://localhost:8181', secure: true, logLevel: 'debug', changeOrigin: true, cookiePathRewrite: { '/datafeeder': '/', }, + headers: { + 'sec-roles': + 'ROLE_SUPERUSER;ROLE_GN_ADMIN;ROLE_ADMINISTRATOR;ROLE_USER;ROLE_MAPSTORE_ADMIN', + 'sec-org': 'myOrg', + 'sec-username': 'testadmin', + 'sec-fetch-user': '?1', + 'sec-proxy': 'true', + referer: 'https://georchestra-127-0-1-1.traefik.me/', + 'sec-orgname': 'myOrg', + }, }, } diff --git a/support-services/datafeeder/datadir/console/cities.geojson b/support-services/datafeeder/datadir/console/cities.geojson new file mode 100644 index 0000000000..b8caaed321 --- /dev/null +++ b/support-services/datafeeder/datadir/console/cities.geojson @@ -0,0 +1,220 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [9.266, 42.666], + [9.27, 42.667], + [9.277, 42.665], + [9.28, 42.665], + [9.283, 42.662], + [9.286, 42.665], + [9.289, 42.665], + [9.295, 42.668], + [9.299, 42.668], + [9.3, 42.669], + [9.306, 42.67], + [9.31, 42.67], + [9.315, 42.668], + [9.317, 42.67], + [9.321, 42.672], + [9.324, 42.67], + [9.326, 42.672], + [9.328, 42.672], + [9.332, 42.675], + [9.334, 42.675], + [9.338, 42.678], + [9.339, 42.681], + [9.339, 42.69], + [9.338, 42.691], + [9.33, 42.693], + [9.329, 42.694], + [9.322, 42.696], + [9.319, 42.693], + [9.317, 42.693], + [9.314, 42.689], + [9.305, 42.684], + [9.3, 42.684], + [9.299, 42.681], + [9.301, 42.68], + [9.299, 42.676], + [9.293, 42.674], + [9.288, 42.674], + [9.283, 42.678], + [9.283, 42.681], + [9.279, 42.684], + [9.28, 42.689], + [9.277, 42.691], + [9.275, 42.694], + [9.272, 42.694], + [9.269, 42.697], + [9.269, 42.701], + [9.266, 42.7], + [9.262, 42.7], + [9.259, 42.701], + [9.259, 42.703], + [9.256, 42.706], + [9.255, 42.712], + [9.257, 42.718], + [9.254, 42.718], + [9.247, 42.72], + [9.245, 42.718], + [9.234, 42.72], + [9.229, 42.718], + [9.234, 42.709], + [9.235, 42.708], + [9.233, 42.704], + [9.237, 42.7], + [9.242, 42.699], + [9.243, 42.701], + [9.245, 42.7], + [9.246, 42.697], + [9.25, 42.697], + [9.251, 42.698], + [9.256, 42.695], + [9.259, 42.694], + [9.262, 42.692], + [9.263, 42.689], + [9.261, 42.687], + [9.258, 42.682], + [9.259, 42.681], + [9.258, 42.678], + [9.261, 42.676], + [9.264, 42.676], + [9.266, 42.672], + [9.266, 42.666] + ] + ] + }, + "properties": { + "NOM_COM_M": "SAINT FLORENT", + "INSEE_COM": "2B298", + "INSEE_DEP": "2B" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [9.065, 41.963], + [9.068, 41.956], + [9.074, 41.952], + [9.073, 41.95], + [9.075, 41.945], + [9.07, 41.94], + [9.07, 41.936], + [9.075, 41.933], + [9.078, 41.933], + [9.083, 41.932], + [9.086, 41.932], + [9.089, 41.928], + [9.088, 41.926], + [9.094, 41.923], + [9.097, 41.927], + [9.102, 41.924], + [9.105, 41.917], + [9.109, 41.918], + [9.109, 41.922], + [9.106, 41.923], + [9.107, 41.928], + [9.109, 41.931], + [9.109, 41.939], + [9.113, 41.942], + [9.113, 41.952], + [9.115, 41.959], + [9.115, 41.963], + [9.108, 41.972], + [9.11, 41.975], + [9.104, 41.976], + [9.102, 41.977], + [9.097, 41.978], + [9.095, 41.978], + [9.091, 41.975], + [9.086, 41.975], + [9.085, 41.972], + [9.083, 41.971], + [9.083, 41.968], + [9.08, 41.966], + [9.077, 41.966], + [9.074, 41.963], + [9.072, 41.963], + [9.07, 41.961], + [9.068, 41.963], + [9.065, 41.963] + ] + ] + }, + "properties": { + "NOM_COM_M": "TASSO", + "INSEE_COM": "2A322", + "INSEE_DEP": "2A" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [9.393, 42.675], + [9.397, 42.673], + [9.404, 42.671], + [9.406, 42.669], + [9.412, 42.667], + [9.424, 42.665], + [9.429, 42.665], + [9.435, 42.662], + [9.443, 42.661], + [9.45, 42.662], + [9.447, 42.669], + [9.446, 42.675], + [9.447, 42.679], + [9.447, 42.685], + [9.446, 42.686], + [9.448, 42.69], + [9.452, 42.691], + [9.453, 42.696], + [9.452, 42.7], + [9.456, 42.706], + [9.455, 42.708], + [9.452, 42.709], + [9.449, 42.711], + [9.446, 42.71], + [9.445, 42.708], + [9.438, 42.706], + [9.436, 42.706], + [9.432, 42.708], + [9.428, 42.708], + [9.423, 42.709], + [9.42, 42.709], + [9.418, 42.707], + [9.413, 42.705], + [9.405, 42.705], + [9.406, 42.703], + [9.403, 42.699], + [9.4, 42.699], + [9.4, 42.695], + [9.398, 42.693], + [9.398, 42.69], + [9.396, 42.689], + [9.396, 42.685], + [9.393, 42.683], + [9.39, 42.676], + [9.393, 42.675] + ] + ] + }, + "properties": { + "NOM_COM_M": "BASTIA", + "INSEE_COM": "2B033", + "INSEE_DEP": "2B" + } + } + ] +} diff --git a/support-services/datafeeder/datadir/console/console.properties b/support-services/datafeeder/datadir/console/console.properties new file mode 100644 index 0000000000..1083b473ba --- /dev/null +++ b/support-services/datafeeder/datadir/console/console.properties @@ -0,0 +1,165 @@ +# set by GeorchestraDatabaseContainer as a System property, +# so bad it looks like console doesn't resolve ${} placeholders +#pgsqlHost=${jdbc.host} +# set by GeorchestraDatabaseContainer as a System property +#pgsqlPort=${jdbc.port} +pgsqlDatabase=georchestra +pgsqlUser=georchestra +pgsqlPassword=georchestra + +# Org type values is used to populate the drop down list from /console/account/new +# default: Association,Company,NGO,Individual,Other +orgTypeValues=Association,Company,NGO,Individual,Other +# Areas map configuration +# This map appears on the /console/account/new page, when the user checks the "my org does not exist" checkbox. +# Currently the map is configured with the EPSG:4326 SRS. +# Center of map +AreaMapCenter=9.3707, 42.0753 + +# Zoom of map +AreaMapZoom=7 + +# AreasUrl is the URL of a static geojson file in the current folder, which +# provides the basic geometries used to build up organization's areas. +# Also accepts an URL, which can be a static file or a WFS request. +# MUST provide a GeoJSON FeatureCollection with the EPSG:4326 SRS. +# example "dynamic" AreasUrl=https://my.server.org/geoserver/ows?SERVICE=WFS&REQUEST=GetFeature&typeName=gadm:gadm_for_countries&outputFormat=json&srs=EPSG:4326&cql_filter=ISO='FRA' or ISO='BEL' +AreasUrl=cities.geojson + +# The following properties are used to configure the map widget behavior: +# AreasKey is the key stored in the org LDAP record to uniquely identify a feature. +AreasKey=INSEE_COM + +# AreasValue is the feature "nice name" which appears in the widget list once selected, and in the search result as well. +AreasValue=NOM_COM_M + +# AreasGroup is the feature property which is used to group together areas. +# eg: if the GeoJSON file represents regions, then AreasGroup might be the property with the "state name". +# CAUTION: AreasGroup **has to** be a string, not a numeric ! +AreasGroup=INSEE_DEP + +# LDAP organizational units + +# Pending users +# default: ou=pendingusers +#pendingUserSearchBaseDN=ou=pendingusers + +# Pending organizations +# default: ou=pendingorgs +#pendingOrgSearchBaseDN=ou=pendingorgs + + +# PostgreSQL database connection parameters + +# Minimum connections pool size +# default: 2 +#dataSource.minPoolSize = 2 + +# Maximum connections pool size +# default: 10 +#dataSource.maxPoolSize = 10 + +# Acquire connection timeout (in ms for c3p0) +# default: 1000 +#dataSource.timeout = 1000 + +# Max time unused connections are kept idle in the pool. Unit is seconds for c3p0. +# default: 60 +#dataSource.maxIdleTime=60 + +# Email-related properties + +# Send emails in HTML format +# default: false +#emailHtml=false + +# Reply-To field in sent emails +# default: ${administratorEmail} +#replyTo=${administratorEmail} + +# From field in sent emails +# default: ${administratorEmail} +#from=${administratorEmail} + +# Subject of email when your account has been created +# default: [${instanceName}] Your account has been created +#subject.account.created=[${instanceName}] Your account has been created + +# Subject of email when your account creation is waiting for validation +# default: [${instanceName}] Your new account is waiting for validation +#subject.account.in.process=[${instanceName}] Your new account is waiting for validation + +# Subject of email for moderator at account creation +# default: [${instanceName}] New account waiting for validation +#subject.requires.moderation=[${instanceName}] New account waiting for validation + +# Subject of email for password change +# default: [${instanceName}] Update your password +#subject.change.password=[${instanceName}] Update your password + +# Subject of email for login change +# default: [${instanceName}] New login for your account +#subject.account.uid.renamed=[${instanceName}] New login for your account + +# Subject of email when a new account has been created +# default: [${instanceName}] New account created +#subject.new.account.notification=[${instanceName}] New account created + +# Encoding of the email templates +# This "é" char should display nicely in a ISO 8859-1 configured editor +# default: UTF-8 +#templateEncoding=UTF-8 + +# Warn a user if their login has been modified +# default: true +#warnUserIfUidModified=true + + +# Email proxy configuration +# Basically, this webapp can send emails on behalf of LDAP users. +# The service endpoint is available at /console/emailProxy +# Usage is restricted to users having the EMAILPROXY role by default, +# cf https://github.com/georchestra/datadir/blob/master/security-proxy/security-mappings.xml +# see https://github.com/georchestra/georchestra/pull/1572 for more information. +# The following restrictions have been implemented to prevent spammers. + +# From field in sent emails +# default: ${administratorEmail} +#emailProxyFromAddress=${administratorEmail} + +# Maximum number of recipients +# default: 10 +#emailProxyMaxRecipient=10 + +# Maximum email body size +# default: 10000 +#emailProxyMaxBodySize=10000 + +# Maximum email subject size +# 200 +#emailProxyMaxSubjectSize=200 + +# Comma-separated list of allowed recipients of emails +# For example: psc@georchestra.org, postmaster@georchestra.org, listmaster@georchestra.org +# default: ${administratorEmail} +#emailProxyRecipientWhitelist=${administratorEmail} + +# Activates SASL +# if set to true, the console will leave the possibility to the administrator +# to set a user to cascade the authentication to another system. +# See https://github.com/georchestra/georchestra/blob/master/docs/tutorials/sasl.md#remote-adldap-authentication-with-sasl +# for more info on how to configure your OpenLDAP to cascade authentication to another LDAP-aware system. +# default: false +#saslEnabled=false + +# name of the remote SASL server +# This option is purely informative, and give hints to the administrator on which server the authentication will take place +# in case of the previous option is activated. +# As all the SASL configuration is made outside of geOrchestra, setting this property won't have influence on the +# server which will be actually queried for authentication. +# default: null +#saslServer=null + +# Activates or disable GDPR-related endpoints +# default: true +#gdpr.allowAccountDeletion=true diff --git a/support-services/datafeeder/datadir/datafeeder/datafeeder.properties b/support-services/datafeeder/datadir/datafeeder/datafeeder.properties new file mode 100644 index 0000000000..90efdc30e4 --- /dev/null +++ b/support-services/datafeeder/datadir/datafeeder/datafeeder.properties @@ -0,0 +1,140 @@ +### The following properties are inherited from the geOrchestra default.properties, +### if you want to override them for datafeeder, uncomment them. +# PostgreSQL server domain name +# Domain name, or IP address, of the PostgreSQL server +# pgsqlHost=localhost + +# PostgreSQL server port +# Listening port of the PostgreSQL server +# pgsqlPort=5432 + +# PostgreSQL database name +# Default common PostgreSQL database for all geOrchestra modules +# pgsqlDatabase=georchestra + +# User to connect to PostgreSQL server +# Default common PostgreSQL user for all geOrchestra modules +# pgsqlUser=georchestra + +# Password to connect to PostgreSQL server +# Default common password of PostgreSQL user for all geOrchestra modules +# pgsqlPassword=georchestra + +#################################### +# Datafeeder specific properties # +#################################### + +publicUrl=http://localhost:8080 +# pgsqlSchema=datafeeder +gn.url=http://geonetwork:8080/geonetwork +gs.url=http://geoserver:8080/geoserver/rest + +# maximum size allowed for uploaded files. (e.g. 128MB, GB can't be used, only KB or MB) +file-upload.max-file-size=5MB +# maximum size allowed for multipart/form-data requests (e.g. 128MB, GB can't be used, only KB or MB) +file-upload.max-request-size=10MB +# size threshold after which files will be written to disk. +file-upload.file-size-threshold=1MB +# directory location where files will be stored by the servlet container once the request exceeds the {@link #fileSizeThreshold} +file-upload.temporary-location=${java.io.tmpdir}/datafeeder/tmp +# directory location where files will be stored. +file-upload.persistent-location=${java.io.tmpdir}/datafeeder/uploads +# select the file to serve as the front-end application configuration +front-end.config.uri=file:${georchestra.datadir}/datafeeder/frontend-config.json + +datafeeder.publishing.geoserver.api-url=${gs.url:http://geoserver:8080/geoserver/rest} +datafeeder.publishing.geoserver.public-url=https://${domainName}/geoserver +datafeeder.publishing.geoserver.layer-client-cache-seconds=3600 +# Use this for HTTP basic authentication to geoserver api url: +#datafeeder.publishing.geoserver.auth.type=basic +#datafeeder.publishing.geoserver.auth.basic.username=geoserver_privileged_user +#datafeeder.publishing.geoserver.auth.basic.password=gerlsSnFd6SmM +# Use this for HTTP-headers based authentication to GeoServer's api url: +datafeeder.publishing.geoserver.auth.type=headers +datafeeder.publishing.geoserver.auth.headers.[sec-proxy]=true +datafeeder.publishing.geoserver.auth.headers.[sec-username]=datafeeder-application +datafeeder.publishing.geoserver.auth.headers.[sec-roles]=ROLE_ADMINISTRATOR + +datafeeder.publishing.geonetwork.api-url=${gn.url:http://geonetwork:8080/geonetwork} +datafeeder.publishing.geonetwork.public-url=https://${domainName}/geonetwork +# Use this for HTTP basic authentication to Geonetwork's api url: +#datafeeder.publishing.geonetwork.auth.type=basic +#datafeeder.publishing.geonetwork.auth.basic.username= +#datafeeder.publishing.geonetwork.auth.basic.password= +# Use this for HTTP-headers based authentication to Geonetwork's api url: +datafeeder.publishing.geonetwork.auth.type=headers +datafeeder.publishing.geonetwork.auth.headers.[sec-proxy]=true +datafeeder.publishing.geonetwork.auth.headers.[sec-username]=testadmin +datafeeder.publishing.geonetwork.auth.headers.[sec-org]=Datafeeder Test +datafeeder.publishing.geonetwork.auth.headers.[sec-roles]=ROLE_ADMINISTRATOR;ROLE_GN_ADMIN +# This is odd, apparently any UUID works as XSRF token, and these two need to be set +datafeeder.publishing.geonetwork.auth.headers.[X-XSRF-TOKEN]=c9f33266-e242-4198-a18c-b01290dce5f1 +datafeeder.publishing.geonetwork.auth.headers.[Cookie]=XSRF-TOKEN=c9f33266-e242-4198-a18c-b01290dce5f1 + +#template-record-id, an existing geonetwork record id to use as template. If provided, takes precedence over template-record +datafeeder.publishing.geonetwork.template-record-id: +#let's use the default template and transform for testing +datafeeder.publishing.geonetwork.template-record: +datafeeder.publishing.geonetwork.template-transform: +#datafeeder.publishing.geonetwork.template-record: file:${georchestra.datadir}/datafeeder/metadata_template.xml +#datafeeder.publishing.geonetwork.template-transform: file:${georchestra.datadir}/datafeeder/metadata_transform.xsl +datafeeder.publishing.geonetwork.defaultResourceType=series + +datafeeder.publishing.backend.local.dbtype=postgis +datafeeder.publishing.backend.local.host=${pgsqlHost} +datafeeder.publishing.backend.local.port=${pgsqlPort} +datafeeder.publishing.backend.local.database=${pgsqlDatabase} +datafeeder.publishing.backend.local.schema=public +datafeeder.publishing.backend.local.user=${pgsqlUser} +datafeeder.publishing.backend.local.passwd=${pgsqlPassword} +datafeeder.publishing.backend.local.preparedStatements=true + +datafeeder.publishing.backend.geoserver.dbtype=postgis +datafeeder.publishing.backend.geoserver.preparedStatements=true +#just a random test property to assert props are loaded from datafeeder.properties +datafeeder.publishing.backend.geoserver.testFromDatafeederPropertiesFile=true +# is a placeholder to be replaced by the actual schema computed from the "sec-org" request header +datafeeder.publishing.backend.geoserver.schema= +#datafeeder.publishing.backend.geoserver.jndiReferenceName=java:comp/env/jdbc/datafeeder +#if a JNDI data source is configured in geoserver, uncomment the above line and comment out the following ones +datafeeder.publishing.backend.geoserver.host=database +datafeeder.publishing.backend.geoserver.port=${pgsqlPort} +datafeeder.publishing.backend.geoserver.database=${pgsqlDatabase} +datafeeder.publishing.backend.geoserver.user=${pgsqlUser} +datafeeder.publishing.backend.geoserver.passwd=${pgsqlPassword} + +# note how to set a property with spaces: property.prefix.[name\ with\ spaces]=value +datafeeder.publishing.backend.geoserver.[Loose\ bbox]=false +datafeeder.publishing.backend.geoserver.[Estimated\ extends]=true + +datafeeder.email.send=false +datafeeder.email.ackTemplate=file:${georchestra.datadir}/datafeeder/templates/analysis-started-email-template.txt +datafeeder.email.analysisFailedTemplate=file:${georchestra.datadir}/datafeeder/templates/analysis-failed-email-template.txt +datafeeder.email.publishFailedTemplate=file:${georchestra.datadir}/datafeeder/templates/data-publishing-failed-email-template.txt +datafeeder.email.publishSuccessTemplate=file:${georchestra.datadir}/datafeeder/templates/data-publishing-succeeded-email-template.txt + +administratorEmail=noreply.georchestra.dev@gmail.com + +# Configuration for SMTP email sending of application events +# Datafeeder will send emails to the user when a job is started, finished, or failed, +# if these configuration properties are set. +spring.mail.host=${smtpHost} +spring.mail.port=${smtpPort} +spring.mail.username=${smtpUser:} +spring.mail.password=${smtpPassword:} +spring.mail.protocol=smtp +spring.mail.test-connection=${smtpTest:false} +spring.mail.properties.mail.smtp.auth=${smtpAuth:false} +spring.mail.properties.mail.smtp.starttls.enable=${smtpTLS:false} + +# Example for gmail: +# make sure 2-step verification is turned off: https://support.google.com/accounts/answer/1064203?hl=en +# and Allow Less Secure App turnes ON: https://myaccount.google.com/lesssecureapps +#spring.mail.host=smtp.gmail.com +#spring.mail.port=587 +#spring.mail.username: noreply.georchestra.dev@gmail.com +#spring.mail.password: ***** +#spring.mail.protocol: smtp +#spring.mail.test-connection: true +#spring.mail.properties.mail.smtp.auth: true +#spring.mail.properties.mail.smtp.starttls.enable: true diff --git a/support-services/datafeeder/datadir/datafeeder/esri.properties b/support-services/datafeeder/datadir/datafeeder/esri.properties new file mode 100644 index 0000000000..34504effca --- /dev/null +++ b/support-services/datafeeder/datadir/datafeeder/esri.properties @@ -0,0 +1 @@ +2154=PROJCS["RGF_1993_Lambert_93",GEOGCS["GCS_RGF_1993",DATUM["D_RGF_1993",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Lambert_Conformal_Conic"],PARAMETER["False_Easting",700000.0],PARAMETER["False_Northing",6600000.0],PARAMETER["Central_Meridian",3.0],PARAMETER["Standard_Parallel_1",49.0],PARAMETER["Standard_Parallel_2",44.0],PARAMETER["Latitude_Of_Origin",46.5],UNIT["Meter",1.0]] \ No newline at end of file diff --git a/support-services/datafeeder/datadir/datafeeder/frontend-config.json b/support-services/datafeeder/datadir/datafeeder/frontend-config.json new file mode 100644 index 0000000000..acccb680d1 --- /dev/null +++ b/support-services/datafeeder/datadir/datafeeder/frontend-config.json @@ -0,0 +1,27 @@ +{ + "encodings": [ + { + "label": "UTF-8", + "value": "UTF-8" + }, + { + "label": "ISO-8859-1", + "value": "ISO-8859-1" + } + ], + "projections": [ + { + "label": "WGS84", + "value": "EPSG:4326" + }, + { + "label": "Lambert 93", + "value": "EPSG:2154" + }, + { + "label": "Web Mercator", + "value": "EPSG:3857" + } + ], + "thesaurusUrl": "https://dev.geo2france.fr/geonetwork/srv/api/registries/vocabularies/search?type=CONTAINS&thesaurus=external.theme.httpinspireeceuropaeutheme-theme&rows=200&q=${q}&uri=**&lang=${lang}" +} diff --git a/support-services/datafeeder/datadir/datafeeder/templates/README.md b/support-services/datafeeder/datadir/datafeeder/templates/README.md new file mode 100644 index 0000000000..b5898dc77a --- /dev/null +++ b/support-services/datafeeder/datadir/datafeeder/templates/README.md @@ -0,0 +1,58 @@ +# Email message templates + +Message template locations are to be configured in `datafeeder.propeties` using the following properties: + +``` +datafeeder.email.ackTemplate=file:${georchestra.datadir}/datafeeder/templates/analysis-started-email-template.txt +datafeeder.email.analysisFailedTemplate=file:${georchestra.datadir}/datafeeder/templates/analysis-failed-email-template.txt +datafeeder.email.publishFailedTemplate=file:${georchestra.datadir}/datafeeder/templates/data-publishing-failed-email-template.txt +datafeeder.email.publishSuccessTemplate=file:${georchestra.datadir}/datafeeder/templates/data-publishing-succeeded-email-template.txt +``` + +A message template consists of a number of one-line header parts, followed by the full message body. + +Variables on a message template are specified using `${variable-name}` notation, like in the following example: + +``` +to: ${user.email} +cc: ${administratorEmail} +bcc: +sender: ${administratorEmail} +from: Georchestra Importer Application +subject: +body: + +Dear ${user.name}, +.... +``` + +The following variables are resolved against the job's user, dataset, or publishing attributes: + +- `${user.name}`: +- `${user.lastName}`: +- `${user.email}`: +- `${job.id}`: +- `${job.createdAt}`: +- `${job.error}`: +- `${job.analizeStatus}`: +- `${job.publishStatus}`: +- `${dataset.name}`: +- `${dataset.featureCount}`: +- `${dataset.encoding}`: +- `${dataset.nativeBounds}`: +- `${publish.tableName}`: +- `${publish.layerName}`: +- `${publish.workspace}`: +- `${publish.srs}`: +- `${publish.encoding}`: +- `${metadata.id}`: +- `${metadata.title}`: +- `${metadata.abstract}`: +- `${metadata.creationDate}`: +- `${metadata.lineage}`: +- `${metadata.latLonBoundingBox}`: +- `${metadata.keywords}`: +- `${metadata.scale}`: + +Additionally, any other ${property} will be resolved against the application context +(for example, any property specified in `default.properties` or `datafeeder.properties`). diff --git a/support-services/datafeeder/datadir/datafeeder/templates/analysis-failed-email-template.txt b/support-services/datafeeder/datadir/datafeeder/templates/analysis-failed-email-template.txt new file mode 100644 index 0000000000..72016e6ef9 --- /dev/null +++ b/support-services/datafeeder/datadir/datafeeder/templates/analysis-failed-email-template.txt @@ -0,0 +1,51 @@ +to: ${user.email} +cc: ${administratorEmail} +bcc: +sender: ${administratorEmail} +from: Georchestra Importer Application +subject: Problem analyzing your ${dataset.name} dataset +body: + +Dear ${user.name}, + +We're sorry to inform you that a problem occurred during the +analysis phase for the ${dataset.name} dataset you have provided +on ${job.createdAt}. + +The platform administrator has been notified too. + +Job identifier: ${job.id} +Reported error: ${job.error} + +You can access the job page at this location: ${publicUrl}/import/${job.id} + +--- +Sent by ${instanceName} + +--- For debugging purposes, full list of available job properties: --- + +user.name: ${user.name} +user.lastName: ${user.lastName} +user.email: ${user.email} +job.id: ${job.id} +job.createdAt: ${job.createdAt} +job.error: ${job.error} +job.analizeStatus: ${job.analizeStatus} +job.publishStatus: ${job.publishStatus} +dataset.name: ${dataset.name} +dataset.featureCount: ${dataset.featureCount} +dataset.encoding: ${dataset.encoding} +dataset.nativeBounds: ${dataset.nativeBounds} +publish.tableName: ${publish.tableName} +publish.layerName: ${publish.layerName} +publish.workspace: ${publish.workspace} +publish.srs: ${publish.srs} +publish.encoding: ${publish.encoding} +metadata.id: ${metadata.id} +metadata.title: ${metadata.title} +metadata.abstract: ${metadata.abstract} +metadata.creationDate: ${metadata.creationDate} +metadata.lineage: ${metadata.lineage} +metadata.latLonBoundingBox: ${metadata.latLonBoundingBox} +metadata.keywords: ${metadata.keywords} +metadata.scale: ${metadata.scale} diff --git a/support-services/datafeeder/datadir/datafeeder/templates/analysis-started-email-template.txt b/support-services/datafeeder/datadir/datafeeder/templates/analysis-started-email-template.txt new file mode 100644 index 0000000000..1788b5d3e5 --- /dev/null +++ b/support-services/datafeeder/datadir/datafeeder/templates/analysis-started-email-template.txt @@ -0,0 +1,43 @@ +to: ${user.email} +cc: ${administratorEmail} +bcc: +sender: ${administratorEmail} +from: Georchestra Importer Application +subject: Analysis process started for your ${dataset.name} dataset +body: + +Dear ${user.name}, + +Your ${dataset.name} dataset was received correctly and is currently being analyzed. + +You can access the job page at this location: ${publicUrl}/import/${job.id} + +--- +Sent by ${instanceName} + +--- For debugging purposes, full list of available job properties: --- +user.name: ${user.name} +user.lastName: ${user.lastName} +user.email: ${user.email} +job.id: ${job.id} +job.createdAt: ${job.createdAt} +job.error: ${job.error} +job.analizeStatus: ${job.analizeStatus} +job.publishStatus: ${job.publishStatus} +dataset.name: ${dataset.name} +dataset.featureCount: ${dataset.featureCount} +dataset.encoding: ${dataset.encoding} +dataset.nativeBounds: ${dataset.nativeBounds} +publish.tableName: ${publish.tableName} +publish.layerName: ${publish.layerName} +publish.workspace: ${publish.workspace} +publish.srs: ${publish.srs} +publish.encoding: ${publish.encoding} +metadata.id: ${metadata.id} +metadata.title: ${metadata.title} +metadata.abstract: ${metadata.abstract} +metadata.creationDate: ${metadata.creationDate} +metadata.lineage: ${metadata.lineage} +metadata.latLonBoundingBox: ${metadata.latLonBoundingBox} +metadata.keywords: ${metadata.keywords} +metadata.scale: ${metadata.scale} diff --git a/support-services/datafeeder/datadir/datafeeder/templates/data-publishing-failed-email-template.txt b/support-services/datafeeder/datadir/datafeeder/templates/data-publishing-failed-email-template.txt new file mode 100644 index 0000000000..4a0b007141 --- /dev/null +++ b/support-services/datafeeder/datadir/datafeeder/templates/data-publishing-failed-email-template.txt @@ -0,0 +1,51 @@ +to: ${user.email} +cc: ${administratorEmail} +bcc: +sender: ${administratorEmail} +from: Georchestra Importer Application +subject: Publishing process failed for your ${dataset.name} dataset +body: + +Dear ${user.name}, + +We're sorry to inform you that the publication process for your +${dataset.name} dataset failed submitted on ${job.createdAt} +has encountered an error and could not finish correctly. + +The platform administrator has been notified too. + +Job identifier: ${job.id} +Reported error: ${job.error} + +You can access the job page at this location: ${publicUrl}/import/${job.id} + +--- +Sent by ${instanceName} + +--- For debugging purposes, full list of available job properties: --- + +user.name: ${user.name} +user.lastName: ${user.lastName} +user.email: ${user.email} +job.id: ${job.id} +job.createdAt: ${job.createdAt} +job.error: ${job.error} +job.analizeStatus: ${job.analizeStatus} +job.publishStatus: ${job.publishStatus} +dataset.name: ${dataset.name} +dataset.featureCount: ${dataset.featureCount} +dataset.encoding: ${dataset.encoding} +dataset.nativeBounds: ${dataset.nativeBounds} +publish.tableName: ${publish.tableName} +publish.layerName: ${publish.layerName} +publish.workspace: ${publish.workspace} +publish.srs: ${publish.srs} +publish.encoding: ${publish.encoding} +metadata.id: ${metadata.id} +metadata.title: ${metadata.title} +metadata.abstract: ${metadata.abstract} +metadata.creationDate: ${metadata.creationDate} +metadata.lineage: ${metadata.lineage} +metadata.latLonBoundingBox: ${metadata.latLonBoundingBox} +metadata.keywords: ${metadata.keywords} +metadata.scale: ${metadata.scale} diff --git a/support-services/datafeeder/datadir/datafeeder/templates/data-publishing-succeeded-email-template.txt b/support-services/datafeeder/datadir/datafeeder/templates/data-publishing-succeeded-email-template.txt new file mode 100644 index 0000000000..75caa8c2cb --- /dev/null +++ b/support-services/datafeeder/datadir/datafeeder/templates/data-publishing-succeeded-email-template.txt @@ -0,0 +1,51 @@ +to: ${user.email} +cc: ${administratorEmail} +bcc: +sender: ${administratorEmail} +from: Georchestra Importer Application +subject: Your ${dataset.name} dataset has been published +body: + +Dear ${user.name}, + +We're pleased to inform you that your ${dataset.name} dataset +submitted on ${job.createdAt} has been correctly processed and published. + +Browse to the following URL to access a map preview of the published layer: + +${publicUrl}/geoserver/${publish.workspace}/wms/reflect?format=application/openlayers&LAYERS=${publish.layerName} + +And to this URL to access its published metadata: + +${publicUrl}/geonetwork/srv/eng/catalog.search#/metadata/${metadata.id} + +--- +Sent by ${instanceName} + +--- For debugging purposes, full list of available job properties: --- + +user.name: ${user.name} +user.lastName: ${user.lastName} +user.email: ${user.email} +job.id: ${job.id} +job.createdAt: ${job.createdAt} +job.error: ${job.error} +job.analizeStatus: ${job.analizeStatus} +job.publishStatus: ${job.publishStatus} +dataset.name: ${dataset.name} +dataset.featureCount: ${dataset.featureCount} +dataset.encoding: ${dataset.encoding} +dataset.nativeBounds: ${dataset.nativeBounds} +publish.tableName: ${publish.tableName} +publish.layerName: ${publish.layerName} +publish.workspace: ${publish.workspace} +publish.srs: ${publish.srs} +publish.encoding: ${publish.encoding} +metadata.id: ${metadata.id} +metadata.title: ${metadata.title} +metadata.abstract: ${metadata.abstract} +metadata.creationDate: ${metadata.creationDate} +metadata.lineage: ${metadata.lineage} +metadata.latLonBoundingBox: ${metadata.latLonBoundingBox} +metadata.keywords: ${metadata.keywords} +metadata.scale: ${metadata.scale} diff --git a/support-services/datafeeder/datadir/default.properties b/support-services/datafeeder/datadir/default.properties new file mode 100644 index 0000000000..7b3627c308 --- /dev/null +++ b/support-services/datafeeder/datadir/default.properties @@ -0,0 +1,27 @@ +domainName=georchestra.test.org +instanceName=geOrchestra +language=en +headerHeight=90 +headerUrl=/header/ +administratorEmail=georchestra@georchestra-127-0-1-1.traefik.me +### PostgreSQL properties +pgsqlHost=database +pgsqlPort=5432 +pgsqlDatabase=georchestra +pgsqlUser=georchestra +pgsqlPassword=georchestra +### LDAP properties +ldapHost=ldap +ldapPort=389 +ldapScheme=ldap +ldapBaseDn=dc=georchestra,dc=org +ldapAdminDn=cn=admin,dc=georchestra,dc=org +ldapAdminPassword=secret +ldapUsersRdn=ou=users +# Roles RDN +ldapRolesRdn=ou=roles +# Organizations RDN +ldapOrgsRdn=ou=orgs +### SMTP properties +smtpHost=smtp +smtpPort=25 diff --git a/support-services/datafeeder/datadir/geonetwork/geonetwork.properties b/support-services/datafeeder/datadir/geonetwork/geonetwork.properties new file mode 100644 index 0000000000..3e04e97dbb --- /dev/null +++ b/support-services/datafeeder/datadir/geonetwork/geonetwork.properties @@ -0,0 +1,23 @@ +georchestra.console.url=http://console:8080 +geonetwork.syncMode=orgs +#geonetwork.syncRolesFilter=.* + +jdbc.host=database +jdbc.port=5432 +jdbc.database=georchestra +jdbc.schema=geonetwork +jdbc.username=georchestra +jdbc.password=georchestra + +geonetwork.profiles.default=RegisteredUser +geonetwork.profiles.rolemappings.[GN_ADMIN]=Administrator +geonetwork.profiles.rolemappings.[GN_REVIEWER]=Reviewer +geonetwork.profiles.rolemappings.[GN_EDITOR]=Editor +geonetwork.profiles.rolemappings.[GN_USER]=RegisteredUser + +geonetwork.scheduled.enabled=true +es.featureproxy.targeturi=http://elasticsearch:9200/gn-features/{_} +es.protocol=http +es.port=9200 +es.host=elasticsearch +es.url=http://elasticsearch:9200 diff --git a/support-services/datafeeder/datadir/geonetwork/log4j/log4j.xml b/support-services/datafeeder/datadir/geonetwork/log4j/log4j.xml new file mode 100644 index 0000000000..67937a7c94 --- /dev/null +++ b/support-services/datafeeder/datadir/geonetwork/log4j/log4j.xml @@ -0,0 +1,200 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/support-services/datafeeder/datadir/kibana/kibana.yml b/support-services/datafeeder/datadir/kibana/kibana.yml new file mode 100644 index 0000000000..65baf31c42 --- /dev/null +++ b/support-services/datafeeder/datadir/kibana/kibana.yml @@ -0,0 +1,5 @@ +server.basePath: '/geonetwork/dashboards' +server.rewriteBasePath: false +kibana.index: '.dashboards' +# https://github.com/elastic/kibana/issues/12918 +server.host: '0.0.0.0' diff --git a/support-services/datafeeder/docker-compose.yml b/support-services/datafeeder/docker-compose.yml new file mode 100644 index 0000000000..7fd96a0409 --- /dev/null +++ b/support-services/datafeeder/docker-compose.yml @@ -0,0 +1,111 @@ +version: '3.1' + +volumes: + datafeeder_uploads: + elasticsearch_data: + +services: + smtp: + image: camptocamp/smtp-sink:latest + ports: + - '2025:25' + restart: always + + database: + image: georchestra/database:latest + environment: + - POSTGRES_USER=georchestra + - POSTGRES_PASSWORD=georchestra + ports: + - '15432:5432' + + ldap: + image: georchestra/ldap:latest + environment: + - SLAPD_ORGANISATION=georchestra + - SLAPD_DOMAIN=georchestra.org + - SLAPD_PASSWORD=secret + - SLAPD_LOG_LEVEL=32768 # See https://www.openldap.org/doc/admin24/slapdconfig.html#loglevel%20%3Clevel%3E + + console: + image: georchestra/console:latest + depends_on: + - ldap + - database + volumes: + - ./datadir:/etc/georchestra + environment: + - JAVA_OPTIONS=-Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF + - XMS=256M + - XMX=1G + ports: + - '38080:8080' + + geoserver: + image: georchestra/geoserver:latest + depends_on: + - ldap + volumes: + - ./datadir:/etc/georchestra + environment: + - JAVA_OPTIONS=-Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF + - XMS=256M + ports: + - '18080:8080' + + geonetwork: + image: georchestra/geonetwork:latest + depends_on: + - console + - database + - elasticsearch + volumes: + - ./datadir:/etc/georchestra + environment: + - DATA_DIR=/var/lib/geonetwork_data + - ES_HOST=elasticsearch + - ES_PROTOCOL=http + - ES_PORT=9200 + - JAVA_OPTIONS=-Duser.home=/tmp/jetty -Dgeorchestra.datadir=/etc/georchestra -Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF + - XMS=256M + ports: + - '8080:8080' + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1 + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + environment: + discovery.type: single-node + ES_JAVA_OPTS: '-Xms1g -Xmx1g' + healthcheck: + test: + [ + 'CMD-SHELL', + 'curl -s -f http://localhost:9200/_cat/health >/dev/null || exit 1', + ] + interval: 30s + timeout: 10s + retries: 10 + volumes: + - elasticsearch_data:/usr/share/elasticsearch/data + + datafeeder: + image: georchestra/datafeeder:latest + depends_on: + database: + condition: service_healthy + volumes: + - ./datadir:/etc/georchestra + - datafeeder_uploads:/tmp/datafeeder + environment: + - JAVA_OPTS=-Xms512m -Xmx512m + # You can set a higher loglevel this way: (ref. https://docs.spring.io/spring-boot/docs/2.1.13.RELEASE/reference/html/boot-features-logging.html#boot-features-custom-log-levels) + - LOGGING_LEVEL_ORG_GEORCHESTRA_DATAFEEDER=INFO + ports: + - 8181:8080 diff --git a/support-services/docker-compose.yml b/support-services/docker-compose.yml index d695e718b3..6debc3c808 100644 --- a/support-services/docker-compose.yml +++ b/support-services/docker-compose.yml @@ -65,28 +65,6 @@ services: ports: - '5601:5601' - # TODO: include the datafeeder backend as well - # datafeeder: - # image: georchestra/datafeeder:latest - # depends_on: - # database: - # condition: service_healthy - # postgis: - # condition: service_healthy - # healthcheck: - # test: ["CMD-SHELL", "curl -s -f http://localhost:8080/datafeeder >/dev/null || exit 1"] - # interval: 30s - # timeout: 10s - # retries: 10 - # volumes: - # - georchestra_datadir:/etc/georchestra - # - datafeeder_uploads:/tmp/datafeeder - # environment: - # - JAVA_OPTS=-Xms512m -Xmx512m - # - LOGGING_LEVEL_ORG_GEORCHESTRA_DATAFEEDER=INFO - # ports: - # - "8081:8080" - geonetwork: image: geonetwork:${GEONETWORK_VERSION} environment: