Merge pull request #1214 from wireapp/release_2020_10_05

Release 2020 10 05

CHANGELOG.md

@@ -1,3 +1,29 @@
+# 2020-10-05
+
+## Release Notes
+
+With this release, the `setCookieDomain` configuration (under `brig`/`config`.`optSettings`) no longer has any effect, and can be removed.
+
+## Security improvements
+
+* Authentication cookies are set to the specific DNS name of the backend server (like nginz-https.example.com), instead of a wildcard domain (like *.example.com). This is achieved by leaving the domain empty in the Set-Cookie header, but changing the code to allow clients with old cookies to continue using them until they get renewed. (#1102)
+
+## Bug Fixes
+
+* Match users on email in SCIM search: Manage invited user by SCIM when SSO is enabled (#1207)
+
+## New Features
+
+* Amount of SFT servers returned on /calls/config/v2 can be limited (default 5, configurable) (#1206)
+* Allow SCIM without SAML (#1200)
+
+## Internal changes
+
+* Cargohold: Log more about AWS errors, ease compatibility testing (#1205, #1210)
+* GHC upgrade to 8.8.4 (#1204)
+* Preparation for APNS notification on iOS 13 devices: Use mutable content for non-voip notifications and update limits (#1212)
+* Cleanup: remove unused scim_user table (#1211)
+
 # 2020-09-04
 
 ## Release Notes

build/alpine/Dockerfile.builder

@@ -1,9 +1,7 @@
 # Requires docker >= 17.05 (requires support for multi-stage builds)
-
-ARG prebuilder_tag=latest
 ARG prebuilder=quay.io/wire/alpine-prebuilder
 
-FROM ${prebuilder}:${prebuilder_tag}
+FROM ${prebuilder}
 WORKDIR /
 
 # Download stack indices and compile/cache dependencies to speed up subsequent
@@ -13,6 +11,7 @@ WORKDIR /
 # a Haddock segfault. See https://github.com/haskell/haddock/issues/928
 
 ARG wire_server_branch=develop
+ARG THREADS=4
 RUN set -x && \
     echo ${wire_server_branch} && \
     git clone -b ${wire_server_branch} https://github.com/wireapp/wire-server.git && \
@@ -21,7 +20,7 @@ RUN set -x && \
     echo "allow-different-user: true" >> /root/.stack/config.yaml && \
     stack build --haddock --dependencies-only haskell-src-exts && \
     stack build --haddock --no-haddock-hyperlink-source haskell-src-exts && \
-    stack build --pedantic --haddock --test --no-run-tests --bench --no-run-benchmarks --dependencies-only && \
+    stack build --pedantic --haddock --test --no-run-tests --bench --no-run-benchmarks --dependencies-only -j${THREADS} && \
     stack install ormolu && \
     cd / && \
     # we run the build only to cache the built source in /root/.stack, we can remove the source code itself

build/alpine/Dockerfile.deps

@@ -1,6 +1,6 @@
 # Requires docker >= 17.05 (requires support for multi-stage builds)
 
-FROM alpine:3.11 as cryptobox-builder
+FROM alpine:3.12 as cryptobox-builder
 
 # compile cryptobox-c
 RUN apk add --no-cache cargo file libsodium-dev git && \
@@ -11,7 +11,7 @@ RUN apk add --no-cache cargo file libsodium-dev git && \
     cargo build --release
 
 # Minimal dependencies for alpine-compiled, dynamically linked wire-server Haskell services
-FROM alpine:3.11
+FROM alpine:3.12
 
 COPY --from=cryptobox-builder /tmp/cryptobox-c/target/release/libcryptobox.so /usr/lib
 

build/alpine/Dockerfile.prebuilder

@@ -1,6 +1,6 @@
 # Requires docker >= 17.05 (requires support for multi-stage builds)
 
-FROM alpine:3.11 as cryptobox-builder
+FROM alpine:3.12 as cryptobox-builder
 
 # compile cryptobox-c
 RUN apk add --no-cache cargo file libsodium-dev git && \
@@ -10,7 +10,7 @@ RUN apk add --no-cache cargo file libsodium-dev git && \
     export SODIUM_USE_PKG_CONFIG=1 && \
     cargo build --release
 
-FROM alpine:3.11
+FROM alpine:3.12
 
 # install cryptobox-c in the new container
 COPY --from=cryptobox-builder /tmp/cryptobox-c/target/release/libcryptobox.so /usr/lib/libcryptobox.so
@@ -22,9 +22,10 @@ RUN apk add --no-cache \
         ca-certificates \
         linux-headers \
         zlib-dev \
-        ghc \
-        ghc-dev \
-        ghc-doc \
+        perl \
+        gmp-dev \
+        libffi-dev \
+        make \
         libsodium-dev \
         openssl-dev \
         protobuf \
@@ -37,10 +38,17 @@ RUN apk add --no-cache \
         libxml2-dev \
         git \
         ncurses \
+        ncurses-dev \
         sed
 
 # get static version of Haskell Stack and use system ghc by default
 ARG STACK_ALPINE_VERSION=2.3.1
 RUN curl -sSfL https://github.com/commercialhaskell/stack/releases/download/v${STACK_ALPINE_VERSION}/stack-${STACK_ALPINE_VERSION}-linux-x86_64-static.tar.gz \
     | tar --wildcards -C /usr/local/bin --strip-components=1 -xzvf - '*/stack' && chmod 755 /usr/local/bin/stack && \
     stack config set system-ghc --global true
+
+ARG GHC_VERSION=8.8.4
+RUN curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org \
+  | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_GHC_VERSION=${GHC_VERSION} sh
+
+ENV PATH=/root/.ghcup/bin:${PATH}

deploy/services-demo/conf/brig.demo-docker.yaml

@@ -102,7 +102,6 @@ optSettings:
   setActivationTimeout: 1209600     # 1 day
   setTeamInvitationTimeout: 1814400 # 21 days
   setUserMaxConnections: 1000
-  setCookieDomain: brig
   setCookieInsecure: false
   setUserCookieRenewAge: 1209600 # 14 days
   setUserCookieLimit: 32

deploy/services-demo/conf/brig.demo.yaml

@@ -102,7 +102,6 @@ optSettings:
   setActivationTimeout: 1209600     # 1 day
   setTeamInvitationTimeout: 1814400 # 21 days
   setUserMaxConnections: 1000
-  setCookieDomain: localhost
   setCookieInsecure: false
   setUserCookieRenewAge: 1209600 # 14 days
   setUserCookieLimit: 32

deploy/services-demo/conf/nginz/nginx.conf

@@ -290,6 +290,11 @@ http {
       proxy_pass http://galley;
     }
 
+    location ~* ^/teams/([^/]*)/legalhold(.*) {
+      include common_response_with_zauth.conf;
+      proxy_pass http://galley;
+    }
+
     # Gundeck Endpoints
 
     rewrite ^/api-docs/push  /push/api-docs?base_url=http://127.0.0.1:8080/ break;

docs/reference/cassandra-schema.cql

@@ -1335,13 +1335,10 @@ CREATE TABLE brig_test.service_prefix (
 
 CREATE KEYSPACE spar_test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
 
-CREATE TABLE spar_test.user (
-    issuer text,
-    sso_id text,
-    uid uuid,
-    PRIMARY KEY (issuer, sso_id)
-) WITH CLUSTERING ORDER BY (sso_id ASC)
-    AND bloom_filter_fp_chance = 0.1
+CREATE TABLE spar_test.scim_external_ids (
+    external text PRIMARY KEY,
+    user uuid
+) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
     AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
@@ -1598,9 +1595,10 @@ CREATE TABLE spar_test.team_provisioning_by_token (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
-CREATE TABLE spar_test.scim_user (
-    id uuid PRIMARY KEY,
-    json blob
+CREATE TABLE spar_test.scim_user_times (
+    uid uuid PRIMARY KEY,
+    created_at timestamp,
+    last_updated_at timestamp
 ) WITH bloom_filter_fp_chance = 0.1
     AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
     AND comment = ''
@@ -1616,3 +1614,24 @@ CREATE TABLE spar_test.scim_user (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
+CREATE TABLE spar_test.user (
+    issuer text,
+    sso_id text,
+    uid uuid,
+    PRIMARY KEY (issuer, sso_id)
+) WITH CLUSTERING ORDER BY (sso_id ASC)
+    AND bloom_filter_fp_chance = 0.1
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.LeveledCompactionStrategy'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+

docs/reference/spar-braindump.md

@@ -21,6 +21,25 @@ documentation answering your questions, look here!
 - if you want to work on our saml/scim implementation and do not have access to [https://github.com/zinfra/backend-issues/issues?q=is%3Aissue+is%3Aopen+label%3Aspar] and [https://github.com/wireapp/design-specs/tree/master/Single%20Sign%20On], please get in touch with us.
 
 
+## design considerations
+
+### SCIM without SAML.
+
+Before https://github.com/wireapp/wire-server/pull/1200, scim tokens could only be added to teams that already had exactly one SAML IdP.  Now, we also allow SAML-less teams to have SCIM provisioning.  This is an alternative to onboarding via team-settings and produces user accounts that are authenticated with email and password.  (Phone may or may not work, but is not officially supported.)
+
+The way this works is different from team-settings: we don't send invites, but we create active users immediately the moment the SCIM user post is processed.  The new thing is that the created user has neither email nor phone nor a SAML identity, nor a password.
+
+How does this work?
+
+**email:** If no SAML IdP is present, SCIM user posts must contain an externalId that is an email address.  This email address is not added to the newly created user, because it has not been validated.  Instead, the flow for changing an email address is triggered in brig: an email is sent to the address containing a validation key, and once the user completes the flow, brig will add the email address to the user.  We had to add very little code for this in this PR, it's all an old feature.
+
+When SCIM user gets are processed, in order to reconstruct the externalId from the user spar is retrieving from brig, we introduce a new json object for the `sso_id` field that looks like this: `{'scim_external_id': '[email protected]'}`.
+
+In order to find users that have email addresses pending validation, we introduce a new table in spar's cassandra called `scim_external_ids`, in analogy to `user`.  We have tried to use brig's internal `GET /i/user&email=...`, but that also finds pending email addresses, and there are corner cases when changing email addresses and waiting for the new address to be validated and the old to be removed...  that made this approach seem infeasible.
+
+**password:** once the user has validated their email address, they need to trigger the "forgot password" flow -- also old code.
+
+
 ## operations
 
 ### enabling / disabling the sso feature for a team

libs/api-bot/src/Network/Wire/Bot/Monad.hs

@@ -94,7 +94,6 @@ import Control.Concurrent.Async
 import Control.Concurrent.STM (retry)
 import Control.Monad.Base
 import Control.Monad.Catch hiding (try)
-import Control.Monad.Fail (MonadFail)
 import Control.Monad.Trans.Control
 import qualified Data.HashMap.Strict as HashMap
 import Data.Id

libs/bilge/src/Bilge/Assert.hs

@@ -38,7 +38,6 @@ import Control.Monad.Writer.Class
 import Control.Monad.Writer.Strict
 import qualified Data.ByteString as S
 import qualified Data.ByteString.Lazy as Lazy
-import Data.List (intersperse, isInfixOf)
 import Imports
 import Network.HTTP.Client
 import System.Console.ANSI

libs/bilge/src/Bilge/IO.hs

@@ -72,7 +72,6 @@ import Bilge.Request
 import Bilge.Response
 import Control.Monad.Base
 import Control.Monad.Catch
-import Control.Monad.Fail (MonadFail)
 import Control.Monad.Trans.Control
 import qualified Data.ByteString.Lazy as LB
 import qualified Data.ByteString.Lazy as Lazy

libs/brig-types/src/Brig/Types/Instances.hs

@@ -33,15 +33,15 @@ instance Cql PrekeyId where
   ctype = Tagged IntColumn
   toCql = CqlInt . fromIntegral . keyId
   fromCql (CqlInt i) = return $ PrekeyId (fromIntegral i)
-  fromCql _ = fail "PrekeyId: Int expected"
+  fromCql _ = Left "PrekeyId: Int expected"
 
 instance Cql ServiceTag where
   ctype = Tagged BigIntColumn
 
   fromCql (CqlBigInt i) = case intToTag i of
     Just t -> return t
-    Nothing -> fail $ "unexpected service tag: " ++ show i
-  fromCql _ = fail "service tag: int expected"
+    Nothing -> Left $ "unexpected service tag: " ++ show i
+  fromCql _ = Left "service tag: int expected"
 
   toCql = CqlBigInt . tagToInt
 
@@ -50,10 +50,10 @@ instance Cql ServiceKeyPEM where
 
   fromCql (CqlBlob b) =
     maybe
-      (fail "service key pem: malformed key")
+      (Left "service key pem: malformed key")
       pure
       (fromByteString' b)
-  fromCql _ = fail "service key pem: blob expected"
+  fromCql _ = Left "service key pem: blob expected"
 
   toCql = CqlBlob . toByteString
 
@@ -74,15 +74,15 @@ instance Cql ServiceKey where
     p <- required "pem"
     case (t :: Int32) of
       0 -> return $! ServiceKey RsaServiceKey s p
-      _ -> fail $ "Unexpected service key type: " ++ show t
+      _ -> Left $ "Unexpected service key type: " ++ show t
     where
       required :: Cql r => Text -> Either String r
       required f =
         maybe
-          (fail ("ServiceKey: Missing required field '" ++ show f ++ "'"))
+          (Left ("ServiceKey: Missing required field '" ++ show f ++ "'"))
           fromCql
           (lookup f fs)
-  fromCql _ = fail "service key: udt expected"
+  fromCql _ = Left "service key: udt expected"
 
   toCql (ServiceKey RsaServiceKey siz pem) =
     CqlUdt

libs/brig-types/src/Brig/Types/Intra.hs

@@ -64,7 +64,7 @@ instance ToJSON AccountStatus where
   toJSON Deleted = String "deleted"
   toJSON Ephemeral = String "ephemeral"
 
-data AccountStatusResp = AccountStatusResp AccountStatus
+data AccountStatusResp = AccountStatusResp {fromAccountStatusResp :: AccountStatus}
 
 instance ToJSON AccountStatusResp where
   toJSON (AccountStatusResp s) = object ["status" .= s]

libs/brig-types/src/Brig/Types/Provider/Tag.hs

@@ -48,7 +48,6 @@ where
 
 import Cassandra.CQL (Cql)
 import Data.Bits
-import Data.List (foldl')
 import Data.Range
 import qualified Data.Set as Set
 import Imports

libs/dns-util/src/Wire/Network/DNS/SRV.hs

@@ -68,15 +68,15 @@ data SrvEntry = SrvEntry
     srvWeight :: !Word16,
     srvTarget :: !SrvTarget
   }
-  deriving (Eq, Show)
+  deriving (Eq, Show, Ord)
 
 data SrvTarget = SrvTarget
   { -- | the hostname on which the service is offered
     srvTargetDomain :: !Domain,
     -- | the port on which the service is offered
     srvTargetPort :: !Word16
   }
-  deriving (Eq, Show)
+  deriving (Eq, Show, Ord)
 
 data SrvResponse
   = SrvNotAvailable

libs/extended/src/Servant/API/Extended.hs

@@ -21,10 +21,8 @@
 -- errors instead of plaintext.
 module Servant.API.Extended where
 
-import Control.Monad.Trans (liftIO)
 import qualified Data.ByteString.Lazy as BL
 import Data.EitherR (fmapL)
-import Data.Maybe (fromMaybe)
 import Data.String.Conversions (cs)
 import Data.Typeable
 import GHC.TypeLits

libs/galley-types/src/Galley/Types.hs

@@ -85,7 +85,6 @@ import Wire.API.Conversation.Typing
 import Wire.API.CustomBackend
 import Wire.API.Event.Conversation
 import Wire.API.Message
-import Wire.API.User (UserIdList (..))
 import Wire.API.User.Client
 
 --------------------------------------------------------------------------------

libs/galley-types/src/Galley/Types/Teams.hs

@@ -138,7 +138,6 @@ import Data.String.Conversions (cs)
 import Imports
 import Wire.API.Event.Team
 import Wire.API.Team
-import Wire.API.Team (NewTeam (..), Team (..), TeamBinding (..))
 import Wire.API.Team.Conversation
 import Wire.API.Team.Feature
 import Wire.API.Team.Member

libs/gundeck-types/src/Gundeck/Types/Common.hs

@@ -57,5 +57,5 @@ instance ToByteString URI where
 instance FromByteString URI where
   parser = takeByteString >>= parse . Bytes.unpack
 
-parse :: Monad m => String -> m URI
+parse :: (Monad m, MonadFail m) => String -> m URI
 parse = maybe (fail "Invalid URI") (return . URI) . Net.parseURI