Merge pull request #1053 from wireapp/release_2020_04_15

.gitignore

@@ -73,9 +73,6 @@ deploy/dockerephemeral/build/airdock_rvm/
 deploy/dockerephemeral/build/dynamodb_local/
 deploy/dockerephemeral/build/smtp/
 
-# Ignore cabal files; use package.yaml instead
-*.cabal
-
 # Avoid storing generated keys
 /deploy/services-demo/resources/turn/secret.txt
 
@@ -85,3 +82,5 @@ deploy/dockerephemeral/build/smtp/
 
 /libs/libzauth/bzauth-c/deb/usr
 
+# Generated hie.yaml
+hie.yaml

.licensure.yml

@@ -0,0 +1,65 @@
+# See https://github.com/chasinglogic/licensure
+# (As of 2020-02-17, version 0.2.0 (96d7fad9201e19e5193bf7435ab5fdf4a6ad4685) is used)
+
+change_in_place: true
+# Regexes which if matched by a file path will always be excluded from
+# getting a license header
+excludes:
+ - \.gitignore
+ - .*lock
+ - \.git/.*
+ - \.licensure\.yml
+ - README.*
+ - LICENSE.*
+ - .*\.(md|rst|txt|yml|yaml)
+ - Cargo.toml
+# Definition of the licenses used on this project and to what files
+# they should apply.
+licenses:
+ # Either a regex or the string "any" to determine to what files this
+ # license should apply. It is common for projects to have files
+ # under multiple licenses or with multiple copyright holders. This
+ # provides the ability to automatically license files correctly
+ # based on their file paths.
+ #
+ # If "any" is provided all files will match this license.
+ - files: .*\.(hs|hsc|rs)
+ ident: AGPL-3.0
+ authors:
+ - name: Wire Swiss GmbH
+ email: [email protected]
+ year: 2020
+ year: 2020
+ name of author: Wire
+
+ auto_template: false
+ template: |
+ This file is part of the Wire Server implementation.
+
+ Copyright (C) [year] Wire Swiss GmbH <[email protected]>
+
+ This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+# Define type of comment characters to apply based on file extensions.
+comments:
+ # The extensions (or singular extension) field defines which file
+ # extensions to apply the commenter to.
+ - columns: 80
+ extensions:
+ - hs
+ - hsc
+ commenter:
+ type: line
+ comment_char: "--"
+ trailing_lines: 1
+ - columns: 80
+ extensions:
+ - rs
+ commenter:
+ type: line
+ comment_char: "//"
+ trailing_lines: 1

CHANGELOG.md

@@ -1,3 +1,43 @@
+# 2020-04-15
+
+## Upgrade steps (IMPORTANT)
+
+1. Update mapping in ElasticSearch (see [./docs/reference/elastic-search.md](./docs/reference/elastic-search.md))
+2. Upgrade brig and the other services as usual
+3. Migrate data in ElasticSearch (see [./docs/reference/elastic-search.md](./docs/reference/elastic-search.md))
+
+## New features
+
+* Allow `brig-index create` to set ES index settings (#1023)
+* Extended team invitations to have name and phone number (#1032)
+* Allow team members to be searched by teammates. (#964)
+* Better defaults for maxKeyLen and maxValueLen (#1034)
+
+## Bug Fixes
+
+* Fix swagger (#1012, #1031)
+* Custom backend lookup by domain is now case-insensitive (#1013)
+
+## Internal Changes
+
+* Federation: resolve opaque IDs at the edges of galley (#1008)
+* Qualify all API imports in Galley (#1006)
+* types-common: write unit tests for Data.Qualified (#1011)
+* Remove subv4 (#1003)
+* Add federation feature flag to brig and galley (#1014)
+* Add hie.yaml (#1024)
+* Improve reproducibility of builds (#1027)
+* Update types of some brig endpoints to be federation-aware (#1013)
+* Bump to lts-14.27 (#1030)
+* Add comments about which endpoints send which events to clients (#1025)
+* Minimize dependencies of all brig binaries (#1035)
+* Federation: Use status 403 for 'not implemented' (#1036)
+* Add endpoint to count team members using ES (#1022)
+* Rename brig's userName to userDisplayName to avoid confusion (#1039)
+* Upgrade to restund 0.4.14 (#1043)
+* Add license headers to all files (#980, #1045)
+* Federation: Link related issue IDs (#1041)
+
 # 2020-03-10
 
 ## New features

Makefile

@@ -48,6 +48,17 @@ formatf:
 formatc:
  ./tools/ormolu.sh -c
 
+# For any Haskell or Rust file that doesn't mention AGPL yet, add a license header.
+# It's your own reponsibility to keep ormolu happy.
+.PHONY: add-license
+add-license:
+ for file in $$(git grep -L "GNU Affero General Public License" | grep '\.hs$$\|\.hsc$$\|\.rs$$'); do \
+ echo "Adding license to $${file}."; \
+ licensure -i $${file}; \
+ done;
+ @echo ""
+ @echo "you most probably want to run 'make formatf' now to keep ormolu happy"
+
 # Clean
 .PHONY: clean
 clean:
@@ -181,3 +192,7 @@ db-reset:
 
 libzauth:
  $(MAKE) -C libs/libzauth install
+
+.PHONY: hie.yaml
+hie.yaml:
+ ./tools/gen-hie-yaml.sh > hie.yaml

docs/developer/dependencies.md

@@ -95,6 +95,24 @@ sudo apt install rustc cargo -y
 curl https://sh.rustup.rs -sSf | sh
 source $HOME/.cargo/env
 ```
+
+## Formatting Haskell files
+
+You need `ormolu` on your PATH, get it with `stack install ormolu`
+
+## Generating license headers
+
+We use [`licensure`](https://github.com/chasinglogic/licensure).
+
+To get it:
+
+```
+git clone https://github.com/chasinglogic/licensure
+cd licensure
+git checkout 96d7fad9201e19e5193bf7435ab5fdf4a6ad4685 # master as of 2020-02-18
+cargo install --path .
+```
+
 ## makedeb
 
 This is a tool to create debian-style binary packages. It is optional, and is only used if you want to install debian-style packages on your debian or ubuntu system.

docs/reference/elastic-search.md

@@ -0,0 +1,69 @@
+# Maintaining ElasticSearch
+
+## Update mapping
+
+```bash
+ES_HOST=<YOUR_HOST>
+ES_PORT=<YOUR_PORT> # default is 9200
+ES_INDEX=<YOUR_INDEX_NAME> # default is directory
+WIRE_VERSION=<VERSION_YOU_ARE_DEPLOYING>
+
+docker run "quay.io/wire/brig-index:$WIRE_VERSION" update-mapping \
+ --elasticsearch-server "http://$ES_HOST:$ES_PORT" \
+ --elasticsearch-index "$ES_INDEX"
+```
+
+Instead of running this in docker, this can also be done by building the `brig-index` binary from `services/brig` and executing it like this:
+
+```bash
+brig-index update-mapping \
+ --elasticsearch-server "http://$ES_HOST:$ES_PORT" \
+ --elasticsearch-index "$ES_INDEX"
+```
+
+## Migrate Data
+
+```bash
+ES_HOST=<YOUR_HOST>
+ES_PORT=<YOUR_PORT> # default is 9200
+ES_INDEX=<YOUR_INDEX_NAME> # default is directory
+BRIG_CASSANDRA_HOST=<YOUR_C*_HOST>
+BRIG_CASSANDRA_PORT=<YOUR_C*_PORT>
+BRIG_CASSANDRA_KEYSPACE=<YOUR_C*_KEYSPACE>
+WIRE_VERSION=<VERSION_YOU_ARE_DEPLOYING>
+
+docker run "quay.io/wire/brig-index:$WIRE_VERSION" migrate-data \
+ --elasticsearch-server "http://$ES_HOST:$ES_PORT" \
+ --elasticsearch-index "$ES_INDEX" \
+ --cassandra-host "$BRIG_CASSANDRA_HOST" \
+ --cassandra-port "$BRIG_CASSANDRA_PORT" \
+ --cassandra-keyspace "$BRIG_CASSANDRA_KEYSPACE"
+```
+
+(Or, as above, you can also do the same thing without docker.)
+
+## Refill ES documents from Cassandra
+
+This is needed if the information we keep in elastic search increases.
+Also update the indices.
+
+```bash
+ES_HOST=<YOUR_HOST>
+ES_PORT=<YOUR_PORT> # default is 9200
+ES_INDEX=<YOUR_INDEX_NAME> # default is directory
+BRIG_CASSANDRA_HOST=<YOUR_C*_HOST>
+BRIG_CASSANDRA_PORT=<YOUR_C*_PORT>
+BRIG_CASSANDRA_KEYSPACE=<YOUR_C*_KEYSPACE>
+WIRE_VERSION=<VERSION_YOU_ARE_DEPLOYING>
+
+docker run "quay.io/wire/brig-index:$WIRE_VERSION" reindex \
+ --elasticsearch-server "http://$ES_HOST:$ES_PORT" \
+ --elasticsearch-index "$ES_INDEX" \
+ --cassandra-host "$BRIG_CASSANDRA_HOST" \
+ --cassandra-port "$BRIG_CASSANDRA_PORT" \
+ --cassandra-keyspace "$BRIG_CASSANDRA_KEYSPACE"
+```
+
+Subcommand `reindex-if-same-or-newer` can be used instead of `reindex`, if you want to recreate the documents in elasticsearch regardless of their version.
+
+(Or, as above, you can also do the same thing without docker.)

libs/api-bot/api-bot.cabal

@@ -0,0 +1,89 @@
+cabal-version: 1.12
+
+-- This file has been generated from package.yaml by hpack version 0.31.2.
+--
+-- see: https://github.com/sol/hpack
+--
+-- hash: ff48dbfe677cdd01dafb24662f6ef4d5e0dd7f04251cf92a8c7ed61a4787e1c0
+
+name: api-bot
+version: 0.4.2
+synopsis: (Internal) API automation around wire-client
+description: (Internal) API automation around the wire-client.
+category: Network
+stability: experimental
+author: Wire Swiss GmbH
+maintainer: Wire Swiss GmbH <[email protected]>
+license: AGPL-3
+license-file: LICENSE
+build-type: Simple
+
+library
+ exposed-modules:
+ Network.Wire.Bot
+ Network.Wire.Bot.Assert
+ Network.Wire.Bot.Cache
+ Network.Wire.Bot.Clients
+ Network.Wire.Bot.Crypto
+ Network.Wire.Bot.Crypto.Glue
+ Network.Wire.Bot.Email
+ Network.Wire.Bot.Metrics
+ Network.Wire.Bot.Monad
+ Network.Wire.Bot.Report
+ Network.Wire.Bot.Report.Text
+ Network.Wire.Bot.Settings
+ other-modules:
+ Paths_api_bot
+ hs-source-dirs:
+ src
+ default-extensions: AllowAmbiguousTypes BangPatterns ConstraintKinds DataKinds DefaultSignatures DerivingStrategies DeriveFunctor DeriveGeneric DeriveLift DeriveTraversable EmptyCase FlexibleContexts FlexibleInstances FunctionalDependencies GADTs InstanceSigs KindSignatures LambdaCase MultiParamTypeClasses MultiWayIf NamedFieldPuns NoImplicitPrelude OverloadedStrings PackageImports PatternSynonyms PolyKinds QuasiQuotes RankNTypes ScopedTypeVariables StandaloneDeriving TemplateHaskell TupleSections TypeApplications TypeFamilies TypeFamilyDependencies TypeOperators UndecidableInstances ViewPatterns
+ ghc-options: -O2 -Wall -fwarn-tabs -optP-Wno-nonportable-include-path
+ build-depends:
+ HaskellNet >=0.5
+ , HaskellNet-SSL >=0.3
+ , aeson >=0.11
+ , ansi-terminal >=0.6
+ , api-client
+ , async >=2.0
+ , attoparsec >=0.13
+ , base >=4.6 && <5
+ , base64-bytestring >=1.0
+ , bilge >=0.21
+ , bytestring >=0.10
+ , bytestring-conversion >=0.2
+ , cereal >=0.5
+ , containers >=0.5
+ , cryptobox-haskell >=0.1.1
+ , cryptonite >=0.17
+ , directory >=1.2
+ , errors >=2.0
+ , exceptions >=0.8
+ , filepath >=1.3
+ , http-client >=0.4
+ , http-types >=0.8
+ , imports
+ , iso639 >=0.1
+ , lens >=4.1
+ , memory >=0.12
+ , metrics-core >=0.1.17
+ , mime >=0.4
+ , monad-control >=1.0
+ , mtl >=2.1
+ , mwc-random >=0.13
+ , optparse-applicative >=0.11
+ , resource-pool >=0.2
+ , scientific >=0.3
+ , semigroups >=0.10
+ , singletons >=0.10
+ , stm >=2.4
+ , text >=0.11
+ , time >=1.5
+ , tinylog >=0.14
+ , transformers >=0.3
+ , transformers-base >=0.4
+ , types-common >=0.11
+ , unliftio-core >=0.1
+ , unordered-containers >=0.2
+ , uuid >=1.3
+ , vector >=0.10
+ default-language: Haskell2010

libs/api-bot/src/Network/Wire/Bot.hs

@@ -1,4 +1,24 @@
-module Network.Wire.Bot (module M) where
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2020 Wire Swiss GmbH <[email protected]>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Network.Wire.Bot
+ ( module M,
+ )
+where
 
 import Network.Wire.Bot.Email as M
 import Network.Wire.Bot.Monad as M

libs/api-bot/src/Network/Wire/Bot/Assert.hs

@@ -1,6 +1,23 @@
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE OverloadedStrings #-}
 
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2020 Wire Swiss GmbH <[email protected]>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
 module Network.Wire.Bot.Assert where
 
 import Data.Id (ConvId, UserId)