Put upgraded js, views, templates, ...

geoportal/config.yaml

@@ -121,6 +121,16 @@ vars:
     arguments: {db: 0, distributed_lock: true, host: '{REDIS_HOST}', port: '{REDIS_PORT}',
       redis_expiration_time: 86400}
     backend: dogpile.cache.redis
+  pds:
+    staging_url:
+    prod_url:
+    fme_token:
+    owncloud_internal_url:
+    owncloud_external_url:
+    owncloud_user:
+    owncloud_password:
+    smtp_server:
+    bcc_address:
   casipo: {bcc_address: null, fme_token: null, owncloud_external_url: null, owncloud_internal_url: null,
     owncloud_password: null, owncloud_user: null, prod_url: null, smtp_server: null,
     staging_url: null}
@@ -306,7 +316,9 @@ vars:
   hooks: {}
   host: '{VISIBLE_WEB_HOST}'
   host_forward_host: []
-  https_proxy: {unauthorized_ips: '10.0.0.0/8, 127.0.0.1'}
+  https_proxy:
+    unauthorized_ips: '10.0.0.0/8, 127.0.0.1'
+    authorized_hosts: 'ws.geoportal.lu, ws.geoportail.lu, wms.inspire.geoportail.lu, wms.inspire.geoportal.lu'
   interfaces: [main]
   interfaces_config:
     main:
@@ -390,6 +402,9 @@ vars:
   ogcproxy_enable: false
   overview_map: {base_layer: basemap_2015_global, show: false}
   package: geoportailv3
+  anf:
+    email:
+    map_id:
   pag: {
     bcc_address: null, file_server: null, fme_token: null, owncloud_external_url: null,
     owncloud_internal_url: null, owncloud_password: null, owncloud_user: null, prod_url: null,

geoportal/geoportailv3_geoportal/__init__.py

@@ -130,6 +130,12 @@ def main(global_config, **settings):
         request_method="GET"
     )
 
+    config.add_route(
+        "generate_symbol_file",
+        "/mymaps/generate_symbol_file",
+        request_method="GET"
+    )
+
     config.add_route(
         "mymaps_getpublicmaps",
         "/mymaps/public_maps",
@@ -281,6 +287,10 @@ def main(global_config, **settings):
         "feedback",
         "/feedback"
     )
+    config.add_route(
+        "feedbackanf",
+        "/feedbackanf"
+    )
     # pag routes
     config.add_route(
         "pag_url",

geoportal/geoportailv3_geoportal/scripts/__init__.py

@@ -0,0 +1,12 @@
+from pyramid.paster import get_app
+from logging.config import fileConfig
+import os
+
+
+def get_session(app_config, app_name):
+    fileConfig(app_config, defaults=os.environ)
+    get_app(app_config, app_name, options=os.environ)
+
+    # must be done afterwe have loaded the project config
+    from c2cgeoportal_commons.models import DBSession
+    return DBSession()

geoportal/geoportailv3_geoportal/scripts/db2es.py

@@ -9,7 +9,8 @@
 from elasticsearch import helpers
 from elasticsearch.helpers import BulkIndexError
 from elasticsearch.exceptions import ConnectionTimeout
-from geoportailv3.lib.search import get_elasticsearch, get_index, ensure_index
+from geoportailv3_geoportal.lib.search import get_elasticsearch, get_index, \
+    ensure_index
 
 """
 Utility functions for importing data into Elasticsearch from database
@@ -61,7 +62,7 @@ def main():
     try:
         opts, args = getopt.getopt(sys.argv[1:], 'ri', ['reset', 'index'])
     except getopt.GetoptError as err:
-        print str(err)
+        print(str(err))
         sys.exit(2)
     index = False
     reset = False
@@ -99,7 +100,7 @@ def main():
                              chunk_size=multiple,
                              raise_on_error=True)
             except (BulkIndexError, ConnectionTimeout) as e:
-                print "\n %s" % e
+                print("\n {}".format(e))
             if not results:
                 statuslog("\n")
                 break

geoportal/geoportailv3_geoportal/scripts/finalize_c2c_23_data_adaptations.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+
+import os
+import os.path
+import argparse
+import warnings
+import transaction
+
+from sqlalchemy import text
+from . import get_session
+
+
+def main():
+    app_config = "production.ini"
+    app_name = "app"
+
+    if not os.path.isfile(app_config):
+        "Cannot find config file: {0!s}".format(app_config) + 0
+
+    # Ignores pyramid deprecation warnings
+    warnings.simplefilter("ignore", DeprecationWarning)
+
+    session = get_session(app_config, app_name)
+    from c2cgeoportal_commons.models import DBSession, main, static
+    from c2cgeoportal_commons.models.main import OGCSERVER_TYPE_MAPSERVER, \
+        OGCSERVER_TYPE_GEOSERVER,OGCSERVER_AUTH_NOAUTH, OGCServer, Metadata, \
+        TreeItem, TreeGroup, LayerGroup, LayerWMS, RestrictionArea, Interface
+    from geoportailv3_geoportal.models import LuxLayerInternalWMS
+
+
+
+    # Restriction area must have a name.
+    for r_area in session.query(RestrictionArea).all():
+      if r_area.name is None:
+        r_area.name = 'restrictionarea-{}'.format(r_area.id)
+        session.add(r_area)
+    ###### select * from geov3.restrictionarea where name is null or name = '';
+    ###### do we need a unique name?
+
+
+    transaction.commit()
+
+    # Create new ogc_server if not already exists based on url in LuxLayerInternalWMS.
+    for luxwms in session.query(LuxLayerInternalWMS).distinct(LuxLayerInternalWMS.url).all():
+      url = luxwms.url
+      if url is not None:
+        type = OGCSERVER_TYPE_GEOSERVER if 'arcgis' in url else OGCSERVER_TYPE_MAPSERVER
+        already_exists = session.query(OGCServer).filter(OGCServer.url == url).one_or_none()
+        if already_exists is None:
+          ogc_server = OGCServer(
+            name=url,
+            description='Generated during migration to GMF 2.3',
+            url=url,
+            type_=type,
+            image_type='image/png',
+            auth=OGCSERVER_AUTH_NOAUTH,
+          )
+          session.add(ogc_server)
+
+    transaction.commit()
+
+
+
+    # Set correct ogc_server for LuxLayerInternalWMS with url.
+    for luxwms in session.query(LuxLayerInternalWMS).all():
+      url = luxwms.url
+      if url is not None:
+        ogc_server = session.query(OGCServer).filter(OGCServer.url == url).one()
+        layer_wms = session.query(LayerWMS).filter(LayerWMS.id == luxwms.id).one()
+        layer_wms.ogc_server_id = ogc_server.id
+        session.add(layer_wms)
+
+    transaction.commit()
+
+
+
+    # End note:
+    # Now the the column 'url' in LuxLayerInternalWMS must be manually removed.
+
+
+if __name__ == "__main__":
+    main()

geoportal/geoportailv3_geoportal/scripts/layers2es.py

@@ -37,11 +37,13 @@
 import yaml
 import requests
 import json
+import os
 from argparse import ArgumentParser
 from pyramid.paster import get_app, bootstrap
 from pyramid.i18n import TranslationStringFactory, make_localizer
 from pyramid.interfaces import ITranslationDirectories
-from geoportailv3.lib.search import get_elasticsearch, get_index, ensure_index
+from geoportailv3_geoportal.lib.search import get_elasticsearch, get_index, \
+    ensure_index
 from elasticsearch import helpers
 from elasticsearch.helpers import BulkIndexError
 from elasticsearch.exceptions import ConnectionTimeout
@@ -126,7 +128,7 @@ def main():
     app_name = options.app_name
     if app_name is None and "#" in app_config:
         app_config, app_name = app_config.split("#", 1)
-    get_app(app_config, name=app_name)
+    get_app(app_config, name=app_name, options=os.environ)
 
     Import(options)
 
@@ -138,29 +140,32 @@ def __init__(self, options):
         self.layers = []
 
         settings = {}
-        with open(".build/config.yaml") as f:
+        with open("config.yaml") as f:
             settings = yaml.load(f)
 
-        self.languages = settings["available_locale_names"]
-        exluded_themes_string = settings["excluded_themes_from_search"]
+        self.languages = settings["vars"]["available_locale_names"]
+        exluded_themes_string = settings["vars"]["excluded_themes_from_search"]
         exluded_themes = []
         if exluded_themes_string is not None:
             exluded_themes = exluded_themes_string.split(",")
 
         # must be done only once we have loaded the project config
-        from c2cgeoportal.models import DBSession, Interface, Theme, Role
+        from c2cgeoportal_commons.models import DBSession
+        from c2cgeoportal_commons.models.main import Interface, Theme, Role
 
         self.session = DBSession()
 
         self._ = {}
         self.metadata_service_url = \
             'http://shop.geoportail.lu/Portail/inspire/webservices/getMD.jsp'
-        registry = bootstrap(self.options.app_config)['registry']
-        request = bootstrap(self.options.app_config)['request']
+
+        with bootstrap(self.options.app_config, options=os.environ) as env:
+            registry = env['registry']
+            request = env['request']
 
         self.es_layer_index = get_index(request) + '_layers'
         self.tdirs = registry.queryUtility(ITranslationDirectories, default=[])
-        self.tsf = TranslationStringFactory('geoportailv3-client')
+        self.tsf = TranslationStringFactory('geoportailv3_geoportal-client')
 
         self.interfaces = self.session.query(Interface).filter(
             Interface.name.in_(options.interfaces)
@@ -231,7 +236,7 @@ def _add_fts(self, item, interface, action, role):
                     'description': '',
                     'metadata_name': ''
                 }
-                for metadata in item.ui_metadata:
+                for metadata in item.metadatas:
                     if metadata.name == 'metadata_id':
                         params = dict(
                             uid=metadata.value,
@@ -240,6 +245,7 @@ def _add_fts(self, item, interface, action, role):
                         try:
                             resp = requests.get(url=self.metadata_service_url,
                                                 params=params)
+                            data = {}
                             try:
                                 data = json.loads(resp.text)
                             except:
@@ -279,7 +285,7 @@ def _add_folder(self, group, interface, role):
         return self._add_group(group, interface, self.options.folders, role)
 
     def _add_group(self, group, interface, export, role):
-        from c2cgeoportal.models import LayerGroup
+        from c2cgeoportal_commons.models.main import LayerGroup
 
         fill = False
         if hasattr(group, 'children'):
@@ -307,7 +313,7 @@ def _layer_visible(self, layer, role):
         return False
 
     def _add_layer(self, layer, interface, role):
-        from c2cgeoportal.models import LayerV1
+        from c2cgeoportal_commons.models.main import LayerV1
 
         if isinstance(layer, LayerV1):
             return False

geoportal/geoportailv3_geoportal/scripts/tooltips2pot.py

@@ -1,15 +1,16 @@
 # -*- coding: utf-8 -*-
-import sqlahelper
+import codecs
 import traceback
 import sys
-import urllib2
+import urllib.request
+import urllib.parse
 import re
 import polib
-import os.path
+import os
 from os import path
 from pyramid.paster import bootstrap
-from urllib import urlencode
 from geojson import loads as geojson_loads
+from . import get_session
 
 
 def _get_url_with_token(url):
@@ -24,10 +25,11 @@ def _get_url_with_token(url):
         token = urllib2.urlopen(tokenurl, None, 15).read()
         return baseurl + "token=" + token
     except:
-        print url
+        print(url)
         traceback.print_exc(file=sys.stdout)
     return None
 
+TIMEOUT=15
 
 def _get_external_data(url, bbox=None, layer=None):
     body = {'f': 'pjson',
@@ -55,13 +57,14 @@ def _get_external_data(url, bbox=None, layer=None):
     separator = "?"
     if url.find(separator) > 0:
         separator = "&"
-    query = '%s%s%s' % (url, separator, urlencode(body))
+    query = '%s%s%s' % (url, separator, urllib.parse.urlencode(body))
 
     try:
-        result = urllib2.urlopen(query, None, 15)
+        print('Requesting %s' % query)
+        result = urllib.request.urlopen(query, None, TIMEOUT)
         content = result.read()
     except:
-        print query
+        print(query)
         traceback.print_exc(file=sys.stdout)
         return []
 
@@ -70,12 +73,12 @@ def _get_external_data(url, bbox=None, layer=None):
     except:
         return []
     if 'fieldAliases' not in esricoll:
-        print("Error with the layer:  %s using query : %s response: %s"
-              % (layer, query, esricoll))
+        print(("Error with the layer:  %s using query : %s response: %s"
+              % (layer, query, esricoll)))
         return []
     else:
         return dict((value, key)
-                    for key, value in esricoll['fieldAliases'].iteritems())
+                    for key, value in esricoll['fieldAliases'].items())
 
 
 def remove_attributes(attributes, attributes_to_remove,
@@ -101,13 +104,11 @@ def is_in_po(po, search):
 
 
 def main():  # pragma: nocover
-    env = bootstrap("development.ini")
-    from geoportailv3.models import LuxGetfeatureDefinition
-
-    package = env["registry"].settings["package"]
-    directory = "%s/locale/" % package
-    destination = path.join(directory, "%s-tooltips.pot" % package)
+    destination = "/tmp/tooltips.pot"
 
+    session = get_session('development.ini', 'app')
+    from c2cgeoportal_commons.models import DBSession, DBSessions
+    from geoportailv3_geoportal.models import LuxGetfeatureDefinition
     if not os.path.isfile(destination):
         po = polib.POFile()
         po.metadata = {
@@ -118,22 +119,21 @@ def main():  # pragma: nocover
     else:
         po = polib.pofile(destination, encoding="utf-8")
 
-    dbsession = sqlahelper.get_session()
-    results = dbsession.query(LuxGetfeatureDefinition).\
+    results = session.query(LuxGetfeatureDefinition).\
         filter(LuxGetfeatureDefinition.remote_template == False).filter(
             LuxGetfeatureDefinition.template.in_
             (['default.html', 'default_table.html', 'feedbackanf.html'])).all()  # noqa
 
     fields = []
+    print("%d results" % len(results))
     for result in results:
-        engine = sqlahelper.get_engine(result.engine_gfi)
+        engine = DBSessions[result.engine_gfi]
         first_row = None
         if result.query is not None and len(result.query) > 0:
             if "SELECT" in result.query.upper():
                 first_row = engine.execute(result.query).first()
             else:
-                first_row =\
-                    engine.execute("SELECT * FROM " + result.query).first()
+                first_row = engine.execute("SELECT * FROM " + result.query).first()
         if result.rest_url is not None and len(result.rest_url) > 0:
             first_row = _get_external_data(
                 result.rest_url,