Merge pull request #1164 from wireapp/release_2020_07_13

CHANGELOG.md

@@ -1,3 +1,35 @@
+# 2020-07-13
+
+## Release Notes
+
+* If you are self-hosting wire on the public internet, consider [changing your brig server config](https://github.com/wireapp/wire-server/blob/49f414add470f4c5e969814a37bc851e26f6d9a7/docs/reference/user/registration.md#blocking-creation-of-personal-users-new-teams-refrestrictregistration).
+* Deploy all services except nginz.
+* No migrations, no restrictions on deployment order.
+
+## New Features
+
+* Restrict user creation in on-prem installations (#1161)
+* Implement active flag in SCIM for user suspension (#1158)
+
+## Bug Fixes
+
+* Fix setting team feature status in Stern/backoffice (#1146)
+* Add missing Swagger models (#1153)
+* docs/reference/elastic-search.md: fix typos (#1154)
+
+## Internal changes
+
+* Federation: Implement ID mapping (galley) (#1134)
+* Tweak cassandra container settings to get it to work on nixos. (#1155)
+* Merge wireapp/subtree-hscim repository under `/libs`, preserving history (#1152)
+* Add link to twilio message ID format (#1150)
+* Run backoffice locally (#1148)
+* Fix services-demo (#1149, #1156)
+* Add missing license headers (#1143)
+* Test sign up with invalid email (#1141)
+* Fix ormolu script (source code pretty-printing) (#1142)
+
+
 # 2020-06-19
 
 ## Release Notes

Makefile

@@ -161,6 +161,7 @@ docker-services:
  $(MAKE) -C services/cannon docker
  $(MAKE) -C services/proxy docker
  $(MAKE) -C services/spar docker
+ $(MAKE) -C tools/stern docker
  $(MAKE) docker-exe-zauth
  $(MAKE) -C services/nginz docker
 

deploy/dockerephemeral/docker-compose.yaml

@@ -97,10 +97,18 @@ services:
  image: julialongtin/cassandra:0.0.9
  ports:
  - "127.0.0.1:9042:9042"
+ ulimits:
+ memlock: 65536
+ nofile: 100000
+ nproc: 32768
  environment:
-# what's present in the jvm.options file by default.
-# - "CS_JAVA_OPTIONS=-Xmx1024M -Xms1024M -Xmn200M"
+ # what's present in the jvm.options file by default:
+ #- "CS_JAVA_OPTIONS=-Xmx1024M -Xms1024M -Xmn200M"
  - "CS_JVM_OPTIONS=-Xmx128M -Xms128M -Xmn50M"
+
+ # on nixos, you also may need to run
+ # sysctl -w vm.max_map_count=1048576
+ # or add that to your `configuration.nix`
  networks:
  - demo_wire
 

deploy/services-demo/conf/brig.demo-docker.yaml

@@ -84,6 +84,9 @@ zauth:
  sessionTokenTimeout: 604800 # 7 days
  accessTokenTimeout: 900 # 15 minutes
  providerTokenTimeout: 604800 # 7 days
+ legalHoldUserTokenTimeout: 4838400 # 56 days
+ legalHoldSessionTokenTimeout: 604800 # 7 days
+ legalHoldAccessTokenTimeout: 900 # 15 minutes
 
 turn:
  serversV2: resources/turn/servers-v2.txt

deploy/services-demo/conf/galley.demo-docker.yaml

@@ -26,6 +26,13 @@ settings:
  maxConvSize: 128
  intraListing: false
  conversationCodeURI: https://cannon/join/
+ concurrentDeletionEvents: 1024
+ deleteConvThrottleMillis: 0
+
+ featureFlags: # see #RefConfigOptions in `/docs/reference`
+ sso: disabled-by-default
+ legalhold: disabled-by-default
+ teamSearchVisibility: disabled-by-default
 
 logLevel: Info
 logNetStrings: false

deploy/services-demo/conf/galley.demo.yaml

@@ -32,6 +32,7 @@ settings:
  featureFlags: # see #RefConfigOptions in `/docs/reference`
  sso: disabled-by-default
  legalhold: disabled-by-default
+ teamSearchVisibility: disabled-by-default
 
 logLevel: Info
 logNetStrings: false

deploy/services-demo/conf/nginz/nginx-docker.conf

@@ -58,8 +58,12 @@ http {
  #
  # Logging
  #
+ # Note sanitized_request:
+ # We allow passing access_token as query parameter for e.g. websockets
+ # However we do not want to log access tokens.
+ #
 
- log_format custom_zeta '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" - $connection $request_time $upstream_response_time $upstream_cache_status $zauth_user $zauth_connection $request_id $proxy_protocol_addr';
+ log_format custom_zeta '$remote_addr - $remote_user [$time_local] "$sanitized_request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for" - $connection $request_time $upstream_response_time $upstream_cache_status $zauth_user $zauth_connection $request_id $proxy_protocol_addr';
  access_log /dev/stdout custom_zeta;
 
  #
@@ -97,6 +101,8 @@ http {
  }
 
 
+ # Docker DNS, required to resolve the references to stern here.
+ resolver 127.0.0.11;
 
  #
  # Locations
@@ -192,6 +198,7 @@ http {
  include common_response_with_zauth.conf;
  proxy_pass http://brig;
  }
+
  # Cargohold Endpoints
 
  rewrite ^/api-docs/assets /assets/api-docs?base_url=http://127.0.0.1:8080/ break;
@@ -272,6 +279,11 @@ http {
  proxy_pass http://galley;
  }
 
+ location ~* ^/teams/([^/]*)/features/([^/]*) {
+ include common_response_with_zauth.conf;
+ proxy_pass http://galley;
+ }
+
  # Gundeck Endpoints
 
  rewrite ^/api-docs/push /push/api-docs?base_url=http://127.0.0.1:8080/ break;
@@ -345,6 +357,28 @@ http {
  proxy_pass http://spar;
  }
 
+ # Stern Endpoints
+
+ # We add a `/stern` suffix to the URL to resolve clashes with non-Stern endpoints.
+ rewrite ^/backoffice/api-docs/stern /stern/api-docs?base_url=http://127.0.0.1:8080/stern/ break;
+
+ location /stern/api-docs {
+ include common_response_no_zauth.conf;
+ # Using a variable instead of plain upstream makes nginx still start up if stern is not there.
+ # https://sandro-keil.de/blog/let-nginx-start-if-upstream-host-is-unavailable-or-down
+ set $stern stern:8091;
+ proxy_pass http://$stern;
+ }
+
+ location /stern {
+ include common_response_no_zauth.conf;
+ # Using a variable instead of plain upstream makes nginx still start up if stern is not there.
+ # https://sandro-keil.de/blog/let-nginx-start-if-upstream-host-is-unavailable-or-down
+ set $stern stern:8091;
+ # The trailing slash matters, as it makes sure the `/stern` prefix is removed.
+ proxy_pass http://$stern/;
+ }
+
  #
  # Swagger Resource Listing
  #
@@ -363,6 +397,24 @@ http {
  more_set_headers 'Access-Control-Allow-Origin: $http_origin';
  }
 
+ #
+ # Back Office Swagger Resource Listing
+ #
+ location /backoffice/api-docs {
+ zauth off;
+ default_type application/json;
+ root conf/nginz/zwagger-ui;
+ index resources.json;
+ if ($request_method = 'OPTIONS') {
+ add_header 'Access-Control-Allow-Methods' "GET, POST, PUT, DELETE, OPTIONS";
+ add_header 'Access-Control-Allow-Headers' "$http_access_control_request_headers, DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type";
+ add_header 'Content-Type' 'text/plain; charset=UTF-8';
+ add_header 'Content-Length' 0;
+ return 204;
+ }
+ more_set_headers 'Access-Control-Allow-Origin: $http_origin';
+ }
+
  # Swagger UI
 
  location /swagger-ui {

deploy/services-demo/conf/nginz/nginx.conf

@@ -354,6 +354,24 @@ http {
  proxy_pass http://spar;
  }
 
+ # Stern Endpoints
+
+ # We add a `/stern` suffix to the URL to resolve clashes with non-Stern endpoints.
+ rewrite ^/backoffice/api-docs/stern  /stern/api-docs?base_url=http://127.0.0.1:8080/stern/ break;
+
+ location /stern/api-docs {
+ include common_response_no_zauth.conf;
+ # We don't use an `upstream` for stern, since running stern is optional.
+ proxy_pass http://127.0.0.1:8091;
+ }
+
+ location /stern {
+ include common_response_no_zauth.conf;
+ # We don't use an `upstream` for stern, since running stern is optional.
+ # The trailing slash matters, as it makes sure the `/stern` prefix is removed.
+ proxy_pass http://127.0.0.1:8091/;
+ }
+
  #
  # Swagger Resource Listing
  #
@@ -372,6 +390,24 @@ http {
  more_set_headers 'Access-Control-Allow-Origin: $http_origin';
  }
 
+ #
+ # Back Office Swagger Resource Listing
+ #
+ location /backoffice/api-docs {
+ zauth off;
+ default_type application/json;
+ root conf/nginz/zwagger-ui;
+ index resources.json;
+ if ($request_method = 'OPTIONS') {
+ add_header 'Access-Control-Allow-Methods' "GET, POST, PUT, DELETE, OPTIONS";
+ add_header 'Access-Control-Allow-Headers' "$http_access_control_request_headers, DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type";
+ add_header 'Content-Type' 'text/plain; charset=UTF-8';
+ add_header 'Content-Length' 0;
+ return 204;
+ }
+ more_set_headers 'Access-Control-Allow-Origin: $http_origin';
+ }
+
  # Swagger UI
 
  location /swagger-ui {

deploy/services-demo/conf/proxy.demo-docker.yaml

@@ -0,0 +1,9 @@
+host: proxy
+port: 8087
+
+httpPoolSize: 1000
+maxConns: 5000
+secretsConfig: resources/proxy.config
+
+logLevel: Info
+logNetStrings: false

deploy/services-demo/conf/stern.demo-docker.yaml

@@ -0,0 +1,28 @@
+stern:
+ host: stern
+ port: 8091
+
+brig:
+ host: brig
+ port: 8082
+
+galley:
+ host: galley
+ port: 8085
+
+gundeck:
+ host: gundeck
+ port: 8086
+
+# Both ibis and galeb should be made optional for
+# installations where these services are not available
+galeb:
+ host: galeb
+ port: 8089
+
+ibis:
+ host: ibis
+ port: 8090
+
+logLevel: Info
+logNetStrings: false

deploy/services-demo/conf/stern.demo.yaml

@@ -0,0 +1,28 @@
+stern:
+ host: 127.0.0.1
+ port: 8091
+
+brig:
+ host: 127.0.0.1
+ port: 8082
+
+galley:
+ host: 127.0.0.1
+ port: 8085
+
+gundeck:
+ host: 127.0.0.1
+ port: 8086
+
+# Both ibis and galeb should be made optional for
+# installations where these services are not available
+galeb:
+ host: 127.0.0.1
+ port: 8089
+
+ibis:
+ host: 127.0.0.1
+ port: 8090
+
+logLevel: Info
+logNetStrings: false

deploy/services-demo/demo.sh

@@ -4,15 +4,19 @@
 
 set -eo pipefail
 
-USAGE="$0 [docker]"
-MODE="$1"
+USAGE="$0 [docker] [--run-backoffice]"
 docker_deployment="false"
-if [ "$MODE" = "docker" ]; then
+if [ "$1" = "docker" ] || [ "$2" = "docker" ] ; then
  docker_deployment="true"
 fi
+run_backoffice="false"
+if [ "$1" = "--run-backoffice" ] || [ "$2" = "--run-backoffice" ] ; then
+ run_backoffice="true"
+fi
 TOP_LEVEL="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 DOCKER_FILE="$SCRIPT_DIR/docker-compose.yaml"
+DOCKER_FILE_BACKOFFICE="$SCRIPT_DIR/docker-compose-backoffice.yaml"
 DIR="${TOP_LEVEL}/services"
 PARENT_PID=$$
 rm -f /tmp/demo.* # remove previous temp files, if any
@@ -32,7 +36,7 @@ function list_descendants () {
 }
 
 function kill_gracefully() {
- pkill "gundeck|brig|galley|cargohold|cannon|spar"
+ pkill "gundeck|brig|galley|cargohold|cannon|spar|stern"
  sleep 1
  kill $(list_descendants $PARENT_PID) &> /dev/null
 }
@@ -83,7 +87,8 @@ function check_prerequisites() {
  && test -f ${DIR}/../dist/cargohold \
  && test -f ${DIR}/../dist/proxy \
  && test -f ${DIR}/../dist/spar \
- && test -f ${DIR}/../dist/nginx \
+ && test -f ${DIR}/../dist/stern \
+ && ( test -f ${DIR}/../dist/nginx || which nix-build ) \
  || { echo "Not all services are compiled. How about you run 'cd ${TOP_LEVEL} && make services' first?"; exit 1; }
  fi
 }
@@ -107,8 +112,18 @@ function run_haskell_service() {
 function run_nginz() {
  colour=$1
  prefix=$([ -w /usr/local ] && echo /usr/local || echo "${HOME}/.wire-dev")
- (cd ${SCRIPT_DIR} && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${prefix}/lib/ ${DIR}/../dist/nginx -p ${SCRIPT_DIR} -c ${SCRIPT_DIR}/conf/nginz/nginx.conf -g 'daemon off;' || kill_all) \
- | sed -e "s/^/$(tput setaf ${colour})[nginz] /" -e "s/$/$(tput sgr0)/" &
+
+ # For nix we dont need LD_LIBRARY_PATH; we link against libzauth directly.
+ # nix-build will put a symlink to ./result with the nginx artifact
+ if which nix-build; then
+ nginz=$(nix-build "${DIR}/../nix" -A nginz --no-out-link )
+ (cd ${SCRIPT_DIR} && ${nginz}/bin/nginx -p ${SCRIPT_DIR} -c ${SCRIPT_DIR}/conf/nginz/nginx.conf -g 'daemon off;' || kill_all) \
+ | sed -e "s/^/$(tput setaf ${colour})[nginz] /" -e "s/$/$(tput sgr0)/" &
+ else
+ prefix=$([ -w /usr/local ] && echo /usr/local || echo "${HOME}/.wire-dev")
+ (cd ${SCRIPT_DIR} && LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${prefix}/lib/ ${DIR}/../dist/nginx -p ${SCRIPT_DIR} -c ${SCRIPT_DIR}/conf/nginz/nginx.conf -g 'daemon off;' || kill_all) \
+ | sed -e "s/^/$(tput setaf ${colour})[nginz] /" -e "s/$/$(tput sgr0)/" &
+ fi
 }
 
 function copy_brig_templates() {
@@ -146,9 +161,16 @@ if [ "$docker_deployment" = "false" ]; then
  run_haskell_service cargohold ${purpleish}
  run_haskell_service proxy ${redish}
  run_haskell_service spar ${orange}
+ if [ "$run_backoffice" = "true" ]; then
+ run_haskell_service stern ${orange}
+ fi
  run_nginz ${blueish}
 else
- docker-compose --file "$DOCKER_FILE" up
+ if [ "$run_backoffice" = "true" ]; then
+ docker-compose --file "$DOCKER_FILE" --file "$DOCKER_FILE_BACKOFFICE" up
+ else
+ docker-compose --file "$DOCKER_FILE" up
+ fi
 fi
 
 sleep 3 # wait a moment for services to start before continuing