Sidechain Fork test setup (#744)

* New docker-compose setup with pumba to introduce network delays and produce a fork
* Running an integration test while that fork happens makes the fork apparent (test fails)
* Use docker buildx to build the artifacts

.dockerignore

@@ -6,5 +6,6 @@ ci/
 docker/
 docs/
 local-setup/
+scripts/
 *.Dockerfile
 Dockerfile

.github/workflows/build_and_test.yml

@@ -55,7 +55,9 @@ jobs:
 #          tags: integritee-worker-builder-${{ github.sha }}
 
       - name: Build Enclave Test Image
-        run:  docker build -t integritee-worker-enclave-test-${{ github.sha }} --target enclave-test -f build.Dockerfile .
+        env:
+          DOCKER_BUILDKIT: 1
+        run: docker build -t integritee-worker-enclave-test-${{ github.sha }} --target enclave-test -f build.Dockerfile .
 #        uses: docker/build-push-action@v2
 #        with:
 #          context: .
@@ -68,6 +70,8 @@ jobs:
         run: docker run --name ${{ env.BUILD_CONTAINER_NAME }} integritee-worker-enclave-test-${{ github.sha }}
 
       - name: Build Cargo Test Image
+        env:
+          DOCKER_BUILDKIT: 1
         run:  docker build -t integritee-worker-ctest-${{ github.sha }} --target cargo-test -f build.Dockerfile .
 #        uses: docker/build-push-action@v2
 #        with:
@@ -81,7 +85,9 @@ jobs:
         run: docker run --rm integritee-worker-ctest-${{ github.sha }}
 
       - name: Build Deployable Image
-        run: docker build --output=type=docker,dest=/tmp/integritee-worker.tar --target=deployed -f build.Dockerfile .
+        env:
+          DOCKER_BUILDKIT: 1
+        run: docker build --output=type=tar,dest=/tmp/integritee-worker.tar --target=deployed-worker -f build.Dockerfile .
 #        uses: docker/build-push-action@v2
 #        with:
 #          context: .

build.Dockerfile

@@ -31,7 +31,9 @@ ENV SGX_MODE SW
 COPY . /root/work/worker/
 WORKDIR /root/work/worker
 
-RUN make
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+	--mount=type=cache,target=/root/work/worker/target \
+	make
 
 
 ### Enclave Test Stage
@@ -51,31 +53,57 @@ WORKDIR /root/work/worker
 
 CMD cargo test
 
+### Base Runner Stage
+##################################################
+FROM ubuntu:20.04 AS runner
+
+RUN apt update && apt install -y libssl-dev iproute2
+
+COPY --from=powerman/dockerize /usr/local/bin/dockerize /usr/local/bin/dockerize
 
-### Deployment stage
+
+### Deployed CLI client
 ##################################################
-FROM ubuntu:20.04 AS deployed
+FROM runner AS deployed-client
 LABEL maintainer="[email protected]"
 
-WORKDIR /usr/local/bin
+ARG SCRIPT_DIR=/usr/local/worker-cli
+ARG LOG_DIR=/usr/local/log
 
-RUN apt update && apt install -y libssl-dev
+ENV SCRIPT_DIR ${SCRIPT_DIR}
+ENV LOG_DIR ${LOG_DIR}
+
+COPY --from=builder /root/work/worker/bin/integritee-cli /usr/local/bin
+COPY ./cli/*.sh /usr/local/worker-cli/
+
+RUN chmod +x /usr/local/bin/integritee-cli ${SCRIPT_DIR}/*.sh
+RUN mkdir ${LOG_DIR}
+
+RUN ldd /usr/local/bin/integritee-cli && \
+	/usr/local/bin/integritee-cli --version
+
+ENTRYPOINT ["/usr/local/bin/integritee-cli"]
 
-COPY --from=builder /opt/sgxsdk/lib64 /opt/sgxsdk/lib64
-COPY --from=builder /root/work/worker/bin/* ./
-COPY --from=powerman/dockerize /usr/local/bin/dockerize /usr/local/bin/dockerize
+
+### Deployed worker service
+##################################################
+FROM runner AS deployed-worker
+LABEL maintainer="[email protected]"
 
 ENV SGX_SDK /opt/sgxsdk
 ENV LD_LIBRARY_PATH "${LD_LIBRARY_PATH}:${SGX_SDK}/lib64"
 
+WORKDIR /usr/local/bin
+
+COPY --from=builder /opt/sgxsdk/lib64 /opt/sgxsdk/lib64
+COPY --from=builder /root/work/worker/bin/* ./
+
+RUN touch spid.txt key.txt
 RUN chmod +x /usr/local/bin/integritee-service
 RUN ls -al /usr/local/bin
 
 # checks
 RUN ldd /usr/local/bin/integritee-service && \
 	/usr/local/bin/integritee-service --version
 
-WORKDIR /root/work/worker/bin
-RUN touch spid.txt key.txt
-
 ENTRYPOINT ["/usr/local/bin/integritee-service"]

cli/demo_sidechain.sh

@@ -20,7 +20,7 @@
 # TEST_BALANCE_RUN is either "first" or "second"
 # if -m file is set, the mrenclave will be read from file
 
-while getopts ":m:p:A:B:t:" opt; do
+while getopts ":m:p:A:B:t:u:W:V:C:" opt; do
     case $opt in
         t)
             TEST=$OPTARG
@@ -37,23 +37,43 @@ while getopts ":m:p:A:B:t:" opt; do
         B)
             WORKER2PORT=$OPTARG
             ;;
+        u)
+            NODEURL=$OPTARG
+            ;;
+        V)
+            WORKER1URL=$OPTARG
+            ;;
+        W)
+            WORKER2URL=$OPTARG
+            ;;
+        C)
+            CLIENT_BIN=$OPTARG
+            ;;
     esac
 done
 
 # using default port if none given as arguments
 NPORT=${NPORT:-9944}
+NODEURL=${NODEURL:-"ws://127.0.0.1"}
+
 WORKER1PORT=${WORKER1PORT:-2000}
+WORKER1URL=${WORKER1URL:-"wss://127.0.0.1"}
+
 WORKER2PORT=${WORKER2PORT:-3000}
+WORKER2URL=${WORKER2URL:-"wss://127.0.0.1"}
+
+CLIENT_BIN=${CLIENT_BIN:-"./../bin/integritee-cli"}
 
-echo "Using node-port ${NPORT}"
-echo "Using trusted-worker-1-port ${WORKER1PORT}"
-echo "Using trusted-worker-2-port ${WORKER2PORT}"
+echo "Using client binary ${CLIENT_BIN}"
+echo "Using node uri ${NODEURL}:${NPORT}"
+echo "Using trusted-worker uri ${WORKER1URL}:${WORKER1PORT}"
+echo "Using trusted-worker-2 uri ${WORKER2URL}:${WORKER2PORT}"
 
 INITIALFUNDS=50000000000
 AMOUNTTRANSFER=20000000000
 
-CLIENTWORKER1="./../bin/integritee-cli -p ${NPORT} -P ${WORKER1PORT}"
-CLIENTWORKER2="./../bin/integritee-cli -p ${NPORT} -P ${WORKER2PORT}"
+CLIENTWORKER1="${CLIENT_BIN} -p ${NPORT} -P ${WORKER1PORT} -u ${NODEURL} -U ${WORKER1URL}"
+CLIENTWORKER2="${CLIENT_BIN} -p ${NPORT} -P ${WORKER2PORT} -u ${NODEURL} -U ${WORKER2URL}"
 
 if [ "$READMRENCLAVE" = "file" ]
 then
@@ -95,8 +115,8 @@ echo "* Second transfer: Send ${AMOUNTTRANSFER} funds from Alice's incognito acc
 $CLIENTWORKER2 trusted --mrenclave ${MRENCLAVE} --direct transfer ${ICGACCOUNTALICE} ${ICGACCOUNTBOB} ${AMOUNTTRANSFER}
 echo ""
 
-echo "* Get balance of Alice's incognito account (on worker 2)"
-ALICE_BALANCE=$(${CLIENTWORKER2} trusted --mrenclave ${MRENCLAVE} balance ${ICGACCOUNTALICE} | xargs)
+echo "* Get balance of Alice's incognito account (on worker 1)"
+ALICE_BALANCE=$(${CLIENTWORKER1} trusted --mrenclave ${MRENCLAVE} balance ${ICGACCOUNTALICE} | xargs)
 echo "$ALICE_BALANCE"
 echo ""
 

docker/README.md

@@ -2,16 +2,51 @@
 
 ## Prerequisite
 
-Make sure you have installed Docker with docker-compose. On Windows, this can be Docker Desktop with WSL 2 integration.
+Make sure you have installed Docker with `docker-compose` (version > `1.25.0`). On Windows, this can be Docker Desktop with WSL 2 integration.
 
 ## Building the Docker containers
 
-Run `docker-compose build` in this folder to build the worker image. This will build the worker from source and tag it in an image called `integritee-worker:dev`.
+Run
+```
+COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build
+```
+in this folder to build the worker image. This will build the worker from source and tag it in an image called `integritee-worker:dev`.
 
 ## Running the docker setup
 
 `docker-compose up` will start all services (node and workers), using the `integritee-worker:dev` images you've built in the previous step.
 
+## Run the integration tests
+Build with
+```
+COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -f docker-compose.yml -f integration-test.yml
+```
+and then run with
+```
+docker-compose -f docker-compose.yml -f integration-test.yml up --exit-code-from sidechain-integration-test
+```
+
+## Run the fork simulator
+The fork simulation uses `pumba` which in turn uses the Linux traffic control (TC). This is only available on Linux hosts, not on Windows with WSL unfortunately.
+Build the docker-compose setup with
+```
+COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose -f docker-compose.yml -f fork-inducer.yml -f integration-test.yml build
+```
+
+This requires the docker BuildKit (docker version >= 18.09) and support for it in docker-compose (version >= 1.25.0)
+
+Run the 2-worker setup with a fork inducer (pumba) that delays the traffic on worker 2
+```
+docker-compose -f docker-compose.yml -f fork-inducer.yml -f integration-test.yml up --exit-code-from sidechain-integration-test
+```
+
+This should show that the integration test fails, because we had an unhandled fork in the sidechain. Clean up the containers after each run with:
+```
+docker-compose -f docker-compose.yml -f fork-inducer.yml -f integration-test.yml down
+```
+
+We need these different compose files to separate the services that we're using. E.g. we want the integration test and fork simulator to be optional. The same could be solved using `profiles` - but that requires a more up-to-date version of `docker-compose`.
+
 ## FAQ
 ### What do I have to do to stop everything properly?
 With `Ctrl-C` you stop the containers and with `docker-compose down` you clean up/remove the containers. Note that `docker-compose down` will also remove any logs docker has saved, since it will remove all the container context.
@@ -23,13 +58,9 @@ You need to re-build the worker image, using `docker-compose build`.
 You can change the environment variable `RUST_LOG=` in the `docker-compose.yml` for each worker individually.
 
 ### The log from the node are quite a nuisance. Why are they all together.
-You can suppress the log output for a container by setting the logging driver. This can be set to either `none` (completely disables all logs), or `local` (no console output, but docker will record the logs) in the `docker-compose.yml`:
+You can suppress the log output for a container by setting the logging driver. This can be set to either `none` (completely disables all logs), or `local` (docker will record the logs, depending on you docker-compose version, it will also log to `stdout`) in the `docker-compose.yml`:
 ```
 logging:
     driver: local
 ```
-Mind the indent. Explanations for all the logging drivers in `docker-compose` can be found [here](https://docs.docker.com/config/containers/logging/local/).
-
-
-
-
+Mind the indent. Explanations for all the logging drivers in `docker-compose` can be found [here](https://docs.docker.com/config/containers/logging/local/).

docker/docker-compose.yml

@@ -1,42 +1,57 @@
-version: "3.3"
+version: "3.4"
 services:
   integritee-node:
     image: "integritee/integritee-node-dev:1.0.10"
+    container_name: integritee-node
+    networks:
+      - integritee-test-network
     ports:
-      - "9944:9944"
-    command: --dev --rpc-methods unsafe --ws-external --rpc-external
+      - "9912:9912"
+    command: --dev --rpc-methods unsafe --ws-external --rpc-external --ws-port 9912
     logging:
-      driver: local
+      driver: none
   integritee-worker-1:
     image: integritee-worker:dev
+    container_name: integritee-worker-1
     build:
       context: ..
       dockerfile: build.Dockerfile
+      target: deployed-worker
     depends_on: ['integritee-node']
     environment:
       - RUST_LOG=warn,ws=warn,sp_io=warn,substrate_api_client=warn,jsonrpsee_ws_client=warn,jsonrpsee_ws_server=warn,enclave_runtime=warn,integritee_service=warn,ita_stf=warn
+    networks:
+      - integritee-test-network
     ports:
-      - "4545:4545"
+      - "4645:4645"
       - "2011:2011"
-      - "2001:2001"
-    entrypoint: "dockerize -wait tcp://integritee-node:9944 -timeout 30s
+      - "2101:2101"
+    entrypoint: "dockerize -wait tcp://integritee-node:9912 -timeout 30s
       /usr/local/bin/integritee-service --clean-reset --ws-external -M integritee-worker-1 -T wss://integritee-worker-1 
-      -u ws://integritee-node -U ws://integritee-worker-1 -P 2011 -w 2001 
+      -u ws://integritee-node -U ws://integritee-worker-1 -P 2011 -w 2101 -p 9912 -h 4645
       run --dev --skip-ra"
     restart: always
   integritee-worker-2:
     image: integritee-worker:dev
+    container_name: integritee-worker-2
     build:
       context: ..
       dockerfile: build.Dockerfile
+      target: deployed-worker
     depends_on: ['integritee-node', 'integritee-worker-1']
     environment:
       - RUST_LOG=warn,ws=warn,sp_io=warn,substrate_api_client=warn,jsonrpsee_ws_client=warn,jsonrpsee_ws_server=warn,enclave_runtime=warn,integritee_service=warn,ita_stf=warn
+    networks:
+      - integritee-test-network
     ports:
       - "2012:2012"
-      - "2002:2002"
-    entrypoint: "dockerize -wait http://integritee-worker-1:4545/is_initialized -timeout 150s 
+      - "2102:2102"
+      - "4646:4646"
+    entrypoint: "dockerize -wait http://integritee-worker-1:4645/is_initialized -timeout 150s 
       /usr/local/bin/integritee-service --clean-reset --ws-external -M integritee-worker-2 -T wss://integritee-worker-2 
-      -u ws://integritee-node -U ws://integritee-worker-2 -P 2012 -w 2002 
+      -u ws://integritee-node -U ws://integritee-worker-2 -P 2012 -w 2102 -p 9912 -h 4646
       run --dev --skip-ra --request-state"
     restart: "no"
+networks:
+  integritee-test-network:
+    driver: bridge

docker/fork-inducer.yml

@@ -0,0 +1,27 @@
+version: "3.4"
+services:
+  worker-ping:
+    image: worker-ping:dev
+    build:
+      context: .
+      dockerfile: ping.Dockerfile
+    depends_on: [ 'integritee-node', 'integritee-worker-1', 'integritee-worker-2' ]
+    networks:
+      - integritee-test-network
+    #entrypoint: "ping integritee-worker-2 | while read pong; do echo \"$$(date): $$pong\"; done"
+    entrypoint: "ping integritee-worker-2"
+  pumba-network-delay:
+    image: integritee-fork-producer:dev
+    build:
+      context: .
+      dockerfile: fork.Dockerfile
+    depends_on: [ 'integritee-node', 'integritee-worker-1', 'integritee-worker-2' ]
+    networks:
+      - integritee-test-network
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    entrypoint: "dockerize -wait http://integritee-worker-2:4646/is_initialized -timeout 250s
+     pumba --interval 3m netem --interface eth0 --duration 30s delay --time 1000 integritee-worker-2"
+networks:
+  integritee-test-network:
+    driver: bridge

docker/fork.Dockerfile

@@ -0,0 +1,26 @@
+# Copyright 2021 Integritee AG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#           http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### Build Pumba image with dockerize
+##################################################
+FROM scratch AS fork-simulator-deployed
+LABEL maintainer="[email protected]"
+
+COPY --from=gaiaadm/pumba /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=gaiaadm/pumba /pumba /usr/local/bin/pumba
+COPY --from=powerman/dockerize /usr/local/bin/dockerize /usr/local/bin/dockerize
+
+ENV PATH "$PATH:/usr/local/bin"
+
+ENTRYPOINT ["/usr/local/bin/dockerize"]