Custom OSM server and editor configuration, based on @majdal's work (h…

…otosm#3564) * grab majd's work on custom osm into a different branch. * export all variables from the config for the backend and frontend, remove extrapolation in the messages * dont osm.getenv twice * lint * fix config references. make docker overrides work locally by building images from local clone * fix url tests. ensure there's no extra & * use quotes on messages.js * add tests to new frontend config variables Co-authored-by: Wille Marcel <[email protected]> Co-authored-by: Majd Al-shihabi <[email protected]>
faebebin · Sep 7, 2020 · e6f9345 · e6f9345
1 parent 509d8d8
commit e6f9345
Show file tree

Hide file tree

Showing 19 changed files with 79 additions and 46 deletions.
diff --git a/backend/config.py b/backend/config.py
@@ -36,6 +36,12 @@ class EnvironmentConfig:
     # A freely definable secret key for connecting the front end with the back end
     SECRET_KEY = os.getenv("TM_SECRET", None)
 
+    # OSM API, Nomimatim URLs
+    OSM_SERVER_URL = os.getenv("OSM_SERVER_URL", "https://www.openstreetmap.org")
+    OSM_NOMINATIM_SERVER_URL = os.getenv(
+        "OSM_NOMINATIM_SERVER_URL", "https://nominatim.openstreetmap.org"
+    )
+
     # Database connection
     POSTGRES_USER = os.getenv("POSTGRES_USER", None)
     POSTGRES_PASSWORD = os.getenv("POSTGRES_PASSWORD", None)
@@ -89,12 +95,12 @@ class EnvironmentConfig:
 
     # Connection to OSM authentification system
     OSM_OAUTH_SETTINGS = {
-        "base_url": "https://www.openstreetmap.org/api/0.6/",
+        "base_url": "{}/api/0.6/".format(OSM_SERVER_URL),
         "consumer_key": os.getenv("TM_CONSUMER_KEY", None),
         "consumer_secret": os.getenv("TM_CONSUMER_SECRET", None),
-        "request_token_url": "https://www.openstreetmap.org/oauth/request_token",
-        "access_token_url": "https://www.openstreetmap.org/oauth/access_token",
-        "authorize_url": "https://www.openstreetmap.org/oauth/authorize",
+        "request_token_url": "{}/oauth/request_token".format(OSM_SERVER_URL),
+        "access_token_url": "{}/oauth/access_token".format(OSM_SERVER_URL),
+        "authorize_url": "{}/oauth/authorize".format(OSM_SERVER_URL),
     }
 
     # Some more definitions (not overridable)

diff --git a/backend/models/postgis/project.py b/backend/models/postgis/project.py
@@ -260,8 +260,8 @@ def set_country_info(self):
 
         centroid = to_shape(self.centroid)
         lat, lng = (centroid.y, centroid.x)
-        url = "https://nominatim.openstreetmap.org/reverse?format=jsonv2&lat={0}&lon={1}".format(
-            lat, lng
+        url = "{0}/reverse?format=jsonv2&lat={1}&lon={2}".format(
+            current_app.config["OSM_NOMINATIM_SERVER_URL"], lat, lng
         )
         try:
             country_info = requests.get(url).json()  # returns a dict

diff --git a/backend/models/postgis/user.py b/backend/models/postgis/user.py
@@ -2,6 +2,7 @@
 from backend import db
 from sqlalchemy import desc, func
 from geoalchemy2 import functions
+from flask import current_app
 from backend.models.dtos.user_dto import (
     UserDTO,
     UserMappedProjectsDTO,
@@ -73,7 +74,7 @@ def missing_maps_profile_url(self):
 
     @property
     def osm_profile_url(self):
-        return f"https://www.openstreetmap.org/user/{self.username}"
+        return f"{current_app.config['OSM_SERVER_URL']}/user/{self.username}"
 
     def create(self):
         """ Creates and saves the current model to the DB """

diff --git a/backend/services/users/osm_service.py b/backend/services/users/osm_service.py
@@ -21,7 +21,9 @@ def get_osm_details_for_user(user_id: int) -> UserOSMDTO:
         :param user_id: user_id in scope
         :raises OSMServiceError
         """
-        osm_user_details_url = f"http://www.openstreetmap.org/api/0.6/user/{user_id}"
+        osm_user_details_url = (
+            f"{current_app.config['OSM_SERVER_URL']}/api/0.6/user/{user_id}"
+        )
         response = requests.get(osm_user_details_url)
 
         if response.status_code != 200:

diff --git a/docker-compose.override.yml b/docker-compose.override.yml
@@ -3,8 +3,8 @@ version: '3.4'
 
 x-backend-config: &backend
   build:
-    context: "./scripts/docker"
-    dockerfile: "Dockerfile.backend"
+    context: "."
+    dockerfile: "./scripts/docker/Dockerfile.backend"
 
 services:
   # Main application
@@ -25,11 +25,8 @@ services:
       dockerfile: "./scripts/docker/Dockerfile.frontend"
       args:
         - TM_APP_API_URL=http://localhost/api
-    # stdin_open: true
-    # volumes:
-    #   - "./frontend/src:/usr/src/app/src"
-    # environment:
-    #   - REACT_APP_API_URL=http://localhost/api
+    volumes:
+      - ".:/usr/src/app"
     labels:
       - traefik.http.routers.frontend.rule=Host(`localhost`)
       - traefik.http.services.frontend.loadbalancer.server.port=80
diff --git a/docs/setup-docker.md b/docs/setup-docker.md
@@ -19,6 +19,8 @@ In order to configure this connection you have to go to `https://www.openstreetm
 
 Afterwards copy the consumer key and secret from OpenStreetMap into your configuration file `tasking-manager.env`, and set the two variables: `TM_CONSUMER_KEY` and `TM_CONSUMER_SECRET`.
 
+**NB**: if you've configured a custom OSM server, make sure that you create the user and oAuth client there. 
+
 **Run the Tasking Manager**
 
 The **easiest way** to run the Tasking Manager requires [Docker](https://docs.docker.com/get-started/) and [Docker Compose](https://docs.docker.com/compose/) to be installed on your system.  Afterwards you'll just need:

diff --git a/example.env b/example.env
@@ -27,6 +27,15 @@ TM_ORG_INSTAGRAM=https://www.instagram.com/example/
 TM_ORG_YOUTUBE=https://www.youtube.com/user/example
 TM_ORG_GITHUB=https://github.com/example/
 
+# Information about the OSM server - Customize your server here
+# By default, it's the public OpenStreetMap.org server
+OSM_SERVER_URL=https://www.openstreetmap.org
+OSM_NOMINATIM_SERVER_URL=https://nominatim.openstreetmap.org
+OSM_REGISTER_URL=https://www.openstreetmap.org/user/new
+ID_EDITOR_URL=https://www.openstreetmap.org/edit?editor=id
+POTLATCH2_EDITOR_URL=https://www.openstreetmap.org/edit?editor=potlatch2
+
+
 # Mapbox access key to display the maps (optional)
 #
 # In order to use the default basemap, you’ll need to set a

diff --git a/frontend/.env.expand b/frontend/.env.expand
@@ -28,3 +28,8 @@ REACT_APP_USER_STATS_API_URL=$TM_USER_STATS_API_URL
 REACT_APP_HOMEPAGE_STATS_API_URL=$TM_HOMEPAGE_STATS_API_URL
 REACT_APP_OSM_CONSUMER_KEY=$TM_CONSUMER_KEY
 REACT_APP_OSM_CONSUMER_SECRET=$TM_CONSUMER_SECRET
+REACT_APP_OSM_SERVER_URL=$OSM_SERVER_URL
+REACT_APP_TM_ORG_NAME=$TM_ORG_NAME
+REACT_APP_OSM_REGISTER_URL=$OSM_REGISTER_URL
+REACT_APP_ID_EDITOR_URL=$ID_EDITOR_URL
+REACT_APP_POTLATCH2_EDITOR_URL=$POTLATCH2_EDITOR_URL
diff --git a/frontend/src/components/editor.js b/frontend/src/components/editor.js
@@ -3,7 +3,7 @@ import { useSelector, useDispatch } from 'react-redux';
 import * as iD from '@hotosm/id';
 import '@hotosm/id/dist/iD.css';
 
-import { OSM_CONSUMER_KEY, OSM_CONSUMER_SECRET } from '../config';
+import { OSM_CONSUMER_KEY, OSM_CONSUMER_SECRET, OSM_SERVER_URL } from '../config';
 
 export default function Editor({ editorRef, setEditorRef, setDisable, comment, presets }) {
   const dispatch = useDispatch();
@@ -54,7 +54,7 @@ export default function Editor({ editorRef, setEditorRef, setDisable, comment, p
 
       let osm = editorRef.connection();
       const auth = {
-        urlroot: 'https://www.openstreetmap.org',
+        urlroot: OSM_SERVER_URL,
         oauth_consumer_key: OSM_CONSUMER_KEY,
         oauth_secret: OSM_CONSUMER_SECRET,
         oauth_token: session.osm_oauth_token,

diff --git a/frontend/src/components/header/messages.js b/frontend/src/components/header/messages.js
@@ -1,5 +1,4 @@
 import { defineMessages } from 'react-intl';
-
 /**
  * Internationalized messages for use on header.
  */
@@ -68,15 +67,15 @@ export default defineMessages({
   proceedOSMPart1: {
     id: 'signup.proceed_osm.text1',
     defaultMessage:
-      'The Tasking Manager works with OpenStreetMap, a collaborative, open-source map of the world. Everything you map on the Tasking Manager is going to be available on OpenStreetMap.',
+      "The Tasking Manager works with OpenStreetMap, a collaborative, open-source map of the world. Everything you map on the Tasking Manager is going to be available on OpenStreetMap.",
   },
   proceedOSMPart2: {
     id: 'signup.proceed_osm.text2',
     defaultMessage: "First, you'll need an account on OpenStreetMap.org in order to get started.",
   },
   proceedOSMLogin: {
     id: 'signup.proceed_osm.login',
-    defaultMessage: 'I already have an OpenStreetMap account',
+    defaultMessage: "I already have an OpenStreetMap account",
   },
   emailPlaceholder: {
     id: 'input.placeholder.email_address',
@@ -120,7 +119,7 @@ export default defineMessages({
   },
   submitProceedOSM: {
     id: 'signup.button.submit_osm',
-    defaultMessage: 'Create OpenStreetMap account',
+    defaultMessage: "Create OpenStreetMap account",
   },
   signUpQuestion: {
     id: 'signup.modal.question',

diff --git a/frontend/src/components/user/content.js b/frontend/src/components/user/content.js
@@ -5,6 +5,7 @@ import { FormattedMessage, FormattedNumber, FormattedRelativeTime } from 'react-
 import { selectUnit } from '@formatjs/intl-utils';
 import { useCopyClipboard } from '@lokibai/react-use-copy-clipboard';
 import ReactPlaceholder from 'react-placeholder';
+import { OSM_SERVER_URL } from '../../config';
 
 import messages from './messages';
 import { MappingIcon, ClipboardIcon } from '../svgIcons';
@@ -86,7 +87,7 @@ export function OSMCard({ username }: Object) {
         <div className="w-100 w-50-ns fl">
           <a
             className="link red pb2"
-            href={`https://www.openstreetmap.org/user/${username}/account`}
+            href={`${OSM_SERVER_URL}/user/${username}/account`}
             target="_blank"
             rel="noopener noreferrer"
           >

diff --git a/frontend/src/components/userDetail/headerProfile.js b/frontend/src/components/userDetail/headerProfile.js
@@ -9,6 +9,7 @@ import { NextMappingLevel } from '../user/settings';
 import { UserOrganisations } from './userTeamsOrgs';
 import { SectionMenu } from '../menu';
 import OsmLogo from '../../assets/img/osm_logo.png';
+import { OSM_SERVER_URL } from '../../config';
 
 const SocialMedia = ({ data }) => {
   const socialMediaItems = ['twitterId', 'facebookId', 'linkedinId'];
@@ -45,7 +46,7 @@ const SocialMedia = ({ data }) => {
         url = 'https://www.linkedin.com/' + value;
         break;
       case 'osm':
-        url = 'https://openstreetmap.org/user/' + value;
+        url = OSM_SERVER_URL + '/user/' + value;
         break;
       default:
         return null;

diff --git a/frontend/src/config/index.js b/frontend/src/config/index.js
@@ -39,6 +39,11 @@ export const MATOMO_ID = process.env.REACT_APP_MATOMO_ID || '';
 export const IMAGE_UPLOAD_SERVICE = process.env.REACT_APP_IMAGE_UPLOAD_API_URL || '';
 export const HOMEPAGE_VIDEO_URL = process.env.REACT_APP_HOMEPAGE_VIDEO_URL || '';
 
+// OSM API and Editor URLs
+export const OSM_SERVER_URL = process.env.REACT_APP_OSM_SERVER_URL || 'https://www.openstreetmap.org';
+export const ID_EDITOR_URL = process.env.REACT_APP_ID_EDITOR_URL || 'https://www.openstreetmap.org/edit?editor=id&';
+export const POTLATCH2_EDITOR_URL = process.env.REACT_APP_POTLATCH2_EDITOR_URL || 'https://www.openstreetmap.org/edit?editor=potlatch2'
+
 export const MAX_FILESIZE = parseInt(process.env.REACT_APP_MAX_FILESIZE) || 1000000; // bytes
 
 export const TASK_COLOURS = {

diff --git a/frontend/src/config/tests/config.test.js b/frontend/src/config/tests/config.test.js
@@ -94,3 +94,15 @@ it('exports DROPZONE_SETTINGS', () => {
   expect(config.DROPZONE_SETTINGS.maxSize).toBe(256000);
   expect(config.DROPZONE_SETTINGS.multiple).toBeFalsy();
 });
+it('exports OSM_REGISTER_URL', () => {
+  expect(config.OSM_REGISTER_URL.startsWith('https://')).toBeTruthy();
+});
+it('exports OSM_SERVER_URL', () => {
+  expect(config.OSM_SERVER_URL.startsWith('https://')).toBeTruthy();
+});
+it('exports ID_EDITOR_URL', () => {
+  expect(config.ID_EDITOR_URL.startsWith('https://')).toBeTruthy();
+});
+it('exports POTLATCH2_EDITOR_URL', () => {
+  expect(config.POTLATCH2_EDITOR_URL.startsWith('https://')).toBeTruthy();
+});
diff --git a/frontend/src/utils/editorsList.js b/frontend/src/utils/editorsList.js
@@ -1,9 +1,11 @@
+import { ID_EDITOR_URL, POTLATCH2_EDITOR_URL } from '../config';
+
 export function getEditors(filterList, customEditor) {
   let editors = [
     {
       label: 'iD Editor',
       value: 'ID',
-      url: 'http://www.openstreetmap.org/edit?editor=id&',
+      url: ID_EDITOR_URL,
     },
     {
       label: 'JOSM',
@@ -13,7 +15,7 @@ export function getEditors(filterList, customEditor) {
     {
       label: 'Potlatch 2',
       value: 'POTLATCH_2',
-      url: 'http://www.openstreetmap.org/edit?editor=potlatch2',
+      url: POTLATCH2_EDITOR_URL,
     },
     {
       label: 'Field Papers',

diff --git a/frontend/src/utils/openEditor.js b/frontend/src/utils/openEditor.js
@@ -1,4 +1,4 @@
-import { API_URL } from '../config';
+import { API_URL, ID_EDITOR_URL, POTLATCH2_EDITOR_URL } from '../config';
 import { getCentroidAndZoomFromSelectedTasks, getSelectedTasksBBox } from './tasksGeometry';
 
 export function openEditor(
@@ -58,7 +58,7 @@ export function getFieldPapersUrl(centroid, zoomLevel) {
 }
 
 export function getPotlatch2Url(centroid, zoomLevel) {
-  return `https://www.openstreetmap.org/edit?editor=potlatch2#map=${[
+  return `${POTLATCH2_EDITOR_URL}#map=${[
     zoomLevel,
     roundToDecimals(centroid[1], 5),
     roundToDecimals(centroid[0], 5),
@@ -68,7 +68,7 @@ export function getPotlatch2Url(centroid, zoomLevel) {
 export function getIdUrl(project, centroid, zoomLevel, selectedTasks, customUrl) {
   const base = customUrl
     ? formatCustomUrl(customUrl)
-    : 'https://www.openstreetmap.org/edit?editor=id&';
+    : `${ID_EDITOR_URL}`;
   let url = base + '#map=' + [zoomLevel, centroid[1], centroid[0]].join('/');
   if (project.changesetComment) {
     url += '&comment=' + encodeURIComponent(project.changesetComment);

diff --git a/frontend/src/utils/tests/editorsList.test.js b/frontend/src/utils/tests/editorsList.test.js
@@ -6,7 +6,7 @@ describe('test getEditors', () => {
       {
         label: 'iD Editor',
         value: 'ID',
-        url: 'http://www.openstreetmap.org/edit?editor=id&',
+        url: 'https://www.openstreetmap.org/edit?editor=id&',
       },
       {
         label: 'JOSM',
@@ -16,7 +16,7 @@ describe('test getEditors', () => {
       {
         label: 'Potlatch 2',
         value: 'POTLATCH_2',
-        url: 'http://www.openstreetmap.org/edit?editor=potlatch2',
+        url: 'https://www.openstreetmap.org/edit?editor=potlatch2',
       },
       {
         label: 'Field Papers',
@@ -31,7 +31,7 @@ describe('test getEditors', () => {
       {
         label: 'iD Editor',
         value: 'ID',
-        url: 'http://www.openstreetmap.org/edit?editor=id&',
+        url: 'https://www.openstreetmap.org/edit?editor=id&',
       },
       {
         label: 'JOSM',
@@ -52,7 +52,7 @@ describe('test getEditors', () => {
       {
         label: 'iD Editor',
         value: 'ID',
-        url: 'http://www.openstreetmap.org/edit?editor=id&',
+        url: 'https://www.openstreetmap.org/edit?editor=id&',
       },
       {
         label: 'JOSM',

diff --git a/scripts/docker/Dockerfile.backend b/scripts/docker/Dockerfile.backend
@@ -3,12 +3,7 @@ FROM alpine/git as base
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
-# Get the Tasking Manager
-ARG branch=master
-RUN git clone --depth=1 git://github.com/hotosm/tasking-manager.git \
-	--branch $branch /usr/src/app
-
-RUN rm -rf frontend/
+ADD . /usr/src/app
 
 FROM python:3.7-alpine
 

diff --git a/scripts/docker/Dockerfile.frontend b/scripts/docker/Dockerfile.frontend
@@ -3,18 +3,14 @@ FROM alpine/git as base
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
-# Get the Tasking Manager
-ARG branch=master
-RUN git clone --depth=1 git://github.com/hotosm/tasking-manager.git \
-	--branch $branch /usr/src/app
-
-RUN rm -rf backend/ migrations/
+ADD . /usr/src/app
 
 FROM tiangolo/node-frontend:10 as build
 
 WORKDIR /usr/src/app
 
-COPY --from=base /usr/src/app/frontend /usr/src/app
+COPY --from=base /usr/src/app/ /usr/src/app
+WORKDIR /usr/src/app/frontend
 
 ## SETUP
 RUN npm install
@@ -25,7 +21,7 @@ ARG TM_APP_API_URL=http://localhost/api
 RUN npm run build
 
 FROM nginx:stable-alpine
-COPY --from=build /usr/src/app/build /usr/share/nginx/html
+COPY --from=build /usr/src/app/frontend/build /usr/share/nginx/html
 COPY --from=build /nginx.conf /etc/nginx/conf.d/default.conf
 EXPOSE 80