From a4b0d23382d89573b2470c7a6d3f8990a0191d68 Mon Sep 17 00:00:00 2001 From: abattagl Date: Thu, 21 Jan 2021 17:25:24 +0100 Subject: [PATCH] bugfixes scripts adapted to the new repo name readme.md --- README.md | 51 ++++++++++++- buildQiot.sh | 15 +++- comp/docker-compose.yaml | 84 +++++++++++++++++++++ comp/init-mongo.js | 1 + datahub/docker-compose.yaml | 35 ++++----- docker.txt | 15 ---- edge-server/application/docker-compose.yaml | 6 +- edge-server/sensors/docker-compose.yaml | 6 +- edge/application/docker-compose.yaml | 22 ++++-- edge/prod/docker-compose.yaml | 8 +- edge/sensors/docker-compose.yaml | 4 +- mongotest/init-mongo.js | 14 ---- mongotest/start.sh | 4 - mongotest/stop.sh | 2 - 14 files changed, 190 insertions(+), 77 deletions(-) create mode 100644 comp/docker-compose.yaml delete mode 100644 docker.txt delete mode 100644 mongotest/init-mongo.js delete mode 100755 mongotest/start.sh delete mode 100755 mongotest/stop.sh diff --git a/README.md b/README.md index 5f9fe36..8e29d50 100644 --- a/README.md +++ b/README.md @@ -1 +1,50 @@ -# qiot-datahub-docker \ No newline at end of file +## qiot-datahub-docker + +A set od docker-compose files to run the QIoT environment on your host machine using a OCI-compliant container engine and tools. +Podman engine is recommendedthus you'll find below the prerequisited and mandatory actions to take before using these compose files: + +# if you want to go with podman: + +`$ sudo dnf install -y podman podman-compose` + +# Set up a dedicated container network for both your edge pods and datahub pods + + +`$ sudo docker network create edge` - Network dedicated to the services running on the edge device +/etc/cni/net.d/qiot-covid19-edge.conflist + +`$ sudo podman network create datahub` - Network dedicated to the services running on the datacenter +/etc/cni/net.d/qiot-covid19-datahub.conflist + +`$ sudo docker network create qiotconnect` - Network dedicated to the interaction between the edge device and the datahub +/etc/cni/net.d/qiot-covid19-edge.conflist + +# Tools to monitor your container engine: + +Docker: Portainer + +`docker run -d --privileged -p 9000:9000 -p 8000:8000 --name portainer --restart always -v /var/run/docker.sock:/var/run/docker.sock -v portainer_data:/data portainer/portainer-ce` + +Podman: Podman Cockpit extension + +`sudo dnf install cockpit-podman -y` + +# Disable SELinux to allow containers to access persistent-volume mappings + +We can edit the `/etc/selinux/config` text file with our persistent setting, either enforcing, permissive, or disabled. By default this file appears as shown below. + +``` +This file controls the state of SELinux on the system. +SELINUX= can take one of these three values: + enforcing - SELinux security policy is enforced. + permissive - SELinux prints warnings instead of enforcing. + disabled - No SELinux policy is loaded. +LINUX=enforcing +ELINUXTYPE= can take one of three two values: + targeted - Targeted processes are protected, + minimum - Modification of targeted policy. Only selected processes are protected. + mls - Multi Level Security protection. +SELINUXTYPE=targeted +``` + +We can simply edit the SELINUX variable between enforcing, permissive, or disabled, as outlined in the comments of the file. After editing the file the changes will not be in place immediately and will only change after system reboot. \ No newline at end of file diff --git a/buildQiot.sh b/buildQiot.sh index ab8a020..8c447c8 100755 --- a/buildQiot.sh +++ b/buildQiot.sh @@ -1,17 +1,28 @@ cd /home/abattagl/git/qiot/datahub/qiot-datahub-storer/qiot-datahub-storer-gas/ +chmod +x build.sh ./build.sh cd /home/abattagl/git/qiot/datahub/qiot-datahub-storer/qiot-datahub-storer-pollution/ +chmod +x build.sh ./build.sh cd /home/abattagl/git/qiot/datahub/qiot-datahub-aggregation/ +chmod +x build.sh +./build.sh +cd /home/abattagl/git/qiot/datahub/qiot-datahub-query/ +chmod +x build.sh +./build.sh +cd /home/abattagl/git/qiot/datahub/qiot-datahub-importer/ +chmod +x build.sh ./build.sh -#cd /home/abattagl/git/qiot/datahub/qiot-datahub-importer/ -#./build.sh cd /home/abattagl/git/qiot/datahub/qiot-datahub-registration/ +chmod +x build.sh ./build.sh cd /home/abattagl/git/qiot/datahub/qiot-datahub-endpoint/qiot-datahub-endpoint-mqtt/ +chmod +x build.sh ./build.sh #cd /home/abattagl/git/qiot/edge/qiot-sensors-emulator/ +#chmod +x build.sh #./build.sh #cd /home/abattagl/git/qiot/edge/qiot-service/ +#chmod +x build.sh #./build.sh diff --git a/comp/docker-compose.yaml b/comp/docker-compose.yaml new file mode 100644 index 0000000..b7ae018 --- /dev/null +++ b/comp/docker-compose.yaml @@ -0,0 +1,84 @@ +version: '3' +services: + activemq: + image: vromero/activemq-artemis:2-alpine-latest + container_name: "activemq" + restart: "no" + networks: + - datahub + - qiotconnect + ports: + - "8161:8161" +# - "61616:61616" +# - "5672:5672" + - "1883:1883" + environment: + ARTEMIS_USERNAME: "quarkus" + ARTEMIS_PASSWORD: "quarkus" +# kafdrop: +# image: obsidiandynamics/kafdrop +# container_name: "kafkadrop" +# restart: "no" +# ports: +# - "9000:9000" +# environment: +# KAFKA_BROKERCONNECT: "kafka:29092" +# JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify" +# depends_on: +# - "kafka" + kafka: + image: obsidiandynamics/kafka + container_name: "kafka" + restart: "no" + ports: + - "2181:2181" + - "9092:9092" + environment: + KAFKA_LISTENERS: "INTERNAL://:29092,EXTERNAL://kafka:9092" + KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka:29092,EXTERNAL://kafka:9092" + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT" + KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" + KAFKA_ZOOKEEPER_SESSION_TIMEOUT: "6000" + KAFKA_RESTART_ATTEMPTS: "10" + KAFKA_RESTART_DELAY: "5" + ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: "0" + mongo: + image: "mongo" + container_name: "mongo" + environment: + MONGO_INITDB_DATABASE: "qiot" + #MONGO_INITDB_ROOT_USERNAME: "root" + #MONGO_INITDB_ROOT_PASSWORD: "root" + volumes: + #- ./init-mongo.js: /docker-entrypoint-initdb.d/init-mongo.js:ro + - ./volumes/mongo:/data/db + ports: + - "27017-27019:27017-27019" +# mongo-express: +# image: mongo-express +# container_name: mongo-express +# environment: +# ME_CONFIG_MONGODB_SERVER: "mongo" +# ME_CONFIG_MONGODB_PORT: "27017" +# #ME_CONFIG_MONGODB_ENABLE_ADMIN: "false" +# #ME_CONFIG_MONGODB_AUTH_DATABASE: "qiot" +# #ME_CONFIG_MONGODB_AUTH_USERNAME: "qiot" +# #ME_CONFIG_MONGODB_AUTH_PASSWORD: "qiot" +# ports: +# - "8081:8081" +# influx: +# image: "quay.io/influxdb/influxdb:v2.0.3" +# container_name: "influx" +# volumes: +# - ./volumes/influxdb2:/root/.influxdbv2 +# ports: +# - "8086:8086" +# - "9999:9999" +networks: + datahub: + external: true + qiotconnect: + external: true + default: + external: + name: datahub \ No newline at end of file diff --git a/comp/init-mongo.js b/comp/init-mongo.js index 2eeddf5..2ed4c44 100644 --- a/comp/init-mongo.js +++ b/comp/init-mongo.js @@ -1,3 +1,4 @@ +#docker exec -it mongoContainer mongo use qiot db.createUser( { diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index e873b24..67b0b08 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -1,9 +1,12 @@ version: '3' services: datahub-registration: - image: quay.io/qiot/qiot-datahub-registration + image: quay.io/qiotcovid19/qiot-datahub-registration:1.0.1 container_name: "datahub-registration" restart: "no" + networks: + - datahub + - qiotconnect ports: - "5016:5016" environment: @@ -11,7 +14,7 @@ services: HTTPPORT: "5016" MONGODBURL: "mongo:27017" datahub-aggregation: - image: quay.io/qiot/qiot-datahub-aggregation + image: quay.io/qiotcovid19/qiot-datahub-aggregation:1.0.1 container_name: "datahub-aggregation" restart: "no" ports: @@ -21,7 +24,7 @@ services: HTTPPORT: "5018" MONGODBURL: "mongo:27017" datahub-query: - image: quay.io/qiot/qiot-datahub-query + image: quay.io/qiotcovid19/qiot-datahub-query:1.0.1 container_name: "datahub-query" restart: "no" ports: @@ -31,29 +34,15 @@ services: HTTPPORT: "5200" MONGODBURL: "mongo:27017" endpoint-mqtt: - image: quay.io/qiot/qiot-datahub-endpoint-mqtt + image: quay.io/qiotcovid19/qiot-datahub-endpoint-mqtt:1.0.1 container_name: "datahub-endpoint-mqtt" restart: "no" environment: QUARKUS_PROFILE: "staging" AMQPHOST: "activemq" MQTTHOST: "activemq" -# gasstreamer: -# image: quay.io/qiot/qiot-datahub-streamer-gas -# container_name: "datahub-streamer-gas" -# restart: "no" -# environment: -# MESSAGINGHOST: "activemq" -# KAFKABOOTSTRAPURL: "kafka:9092" -# pollutionstreamer: -# image: quay.io/qiot/qiot-datahub-streamer-pollution -# container_name: "datahub-streamer-pollution" -# restart: "no" -# environment: -# MESSAGINGHOST: "activemq" -# KAFKABOOTSTRAPURL: "kafka:9092" gasstorer: - image: quay.io/qiot/qiot-datahub-storer-gas + image: quay.io/qiotcovid19/qiot-datahub-storer-gas:1.0.1 container_name: "datahub-storer-gas" restart: "no" environment: @@ -61,7 +50,7 @@ services: KAFKABOOTSTRAPURL: "kafka:9092" MONGODBURL: "mongo:27017" pollutionstorer: - image: quay.io/qiot/qiot-datahub-storer-pollution + image: quay.io/qiotcovid19/qiot-datahub-storer-pollution:1.0.1 container_name: "datahub-storer-pollution" restart: "no" environment: @@ -69,6 +58,10 @@ services: KAFKABOOTSTRAPURL: "kafka:9092" MONGODBURL: "mongo:27017" networks: + datahub: + external: true + qiotconnect: + external: true default: external: - name: qiot \ No newline at end of file + name: datahub \ No newline at end of file diff --git a/docker.txt b/docker.txt deleted file mode 100644 index 9527f8c..0000000 --- a/docker.txt +++ /dev/null @@ -1,15 +0,0 @@ -docker network create bridge qiot -#ActiveMQ Artemis -docker run -it --rm --net qiot -p 8161:8161 -p 61616:61616 -p 5672:5672 -p 1883:1883 -e ARTEMIS_USERNAME=quarkus -e ARTEMIS_PASSWORD=quarkus --name activemq vromero/activemq-artemis:2-alpine-latest -#Sensors -docker run -it --rm --net qiot -p 5000:5000 --name sensorsemulator quay.io/qiot/qiot-sensors-emulator -#Edge service -docker run -it --rm --net qiot -e SENSORSURL=http://sensorsemulator:5000/ -e MQTTHOST=activemq --name edge-service quay.io/qiot/qiot-service -#Endpoints -docker run -it --rm --net qiot -e MQTTHOST=activemq -e AMQPHOST=activemq --name endpoint-mqtt quay.io/qiot/qiot-datahub-endpoint-mqtt -#Streamers -docker run -it --rm --net qiot -e MESSAGINGHOST=activemq -e KAFKABOOTSTRAPURL=kafka:9092 --name gasstreamer quay.io/qiot/qiot-datahub-streamer-gas -docker run -it --rm --net qiot -e MESSAGINGHOST=activemq -e KAFKABOOTSTRAPURL=kafka:9092 --name pollutionstreamer quay.io/qiot/qiot-datahub-streamer-pollution - - - diff --git a/edge-server/application/docker-compose.yaml b/edge-server/application/docker-compose.yaml index d723950..cd90211 100644 --- a/edge-server/application/docker-compose.yaml +++ b/edge-server/application/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3' services: edge-integrator: - image: quay.io/qiot/qiot-edge-integrator:1-x86_64 + image: quay.io/qiotcovid19/qiot-edge-integrator:1-x86_64 container_name: "edge-integrator" restart: "no" environment: @@ -11,7 +11,7 @@ services: AMQPPASSWORD: "qiot" SENSORSURL: "http://edge-sensors:5000/" edge-server-01: - image: quay.io/qiot/qiot-edge-server:1-x86_64 + image: quay.io/qiotcovid19/qiot-edge-server:1-x86_64 container_name: "edge-server-01" restart: "no" ports: @@ -32,4 +32,4 @@ services: networks: default: external: - name: qiot \ No newline at end of file + name: edge \ No newline at end of file diff --git a/edge-server/sensors/docker-compose.yaml b/edge-server/sensors/docker-compose.yaml index 665181c..6d187dc 100644 --- a/edge-server/sensors/docker-compose.yaml +++ b/edge-server/sensors/docker-compose.yaml @@ -18,7 +18,7 @@ services: expose: - "5672" edge-sensors: - image: quay.io/qiot/qiot-sensors-emulator + image: quay.io/qiotcovid19/qiot-sensors-emulator container_name: "edge-sensors" restart: "no" ports: @@ -26,7 +26,7 @@ services: environment: QUARKUS_PROFILE: "staging" edge-sensors: - image: quay.io/qiot/qiot-sensors-emulator + image: quay.io/qiotcovid19/qiot-sensors-emulator container_name: "edge-sensors" restart: "no" ports: @@ -36,4 +36,4 @@ services: networks: default: external: - name: qiot \ No newline at end of file + name: edge \ No newline at end of file diff --git a/edge/application/docker-compose.yaml b/edge/application/docker-compose.yaml index de0a960..61f5683 100644 --- a/edge/application/docker-compose.yaml +++ b/edge/application/docker-compose.yaml @@ -1,9 +1,12 @@ version: '3' services: edge-service-01: - image: quay.io/qiot/qiot-service:1-x86_64 + image: quay.io/qiotcovid19/edge-service:1.2.0-x86_64 container_name: "edge-service-01" restart: "no" + networks: + - edge + - qiotconnect ports: - "8091:8080" environment: @@ -17,9 +20,12 @@ services: ENABLESSL: "false" MQTTPORT: "1883" edge-service-02: - image: quay.io/qiot/qiot-service:1-x86_64 + image: quay.io/qiotcovid19/edge-service:1.2.0-x86_64 container_name: "edge-service-02" restart: "no" + networks: + - edge + - qiotconnect ports: - "8092:8080" environment: @@ -33,9 +39,12 @@ services: ENABLESSL: "false" MQTTPORT: "1883" edge-service-03: - image: quay.io/qiot/qiot-service:1-x86_64 + image: quay.io/qiotcovid19/edge-service:1.2.0-x86_64 container_name: "edge-service-03" restart: "no" + networks: + - edge + - qiotconnect ports: - "8093:8080" environment: @@ -49,6 +58,7 @@ services: ENABLESSL: "false" MQTTPORT: "1883" networks: - default: - external: - name: qiot \ No newline at end of file + edge: + external: true + qiotconnect: + external: true \ No newline at end of file diff --git a/edge/prod/docker-compose.yaml b/edge/prod/docker-compose.yaml index 3a1e6cf..e04f8df 100644 --- a/edge/prod/docker-compose.yaml +++ b/edge/prod/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3' services: edge-service-prod-01: - image: quay.io/qiot/qiot-service + image: quay.io/qiotcovid19/edge-service:1.2.0 container_name: "edge-service-prod-01" restart: "no" ports: @@ -16,7 +16,7 @@ services: #ENABLESSL: "false" #MQTTPORT: "1883" edge-service-prod-02: - image: quay.io/qiot/qiot-service + image: quay.io/qiotcovid19/edge-service:1.2.0 container_name: "edge-service-prod-02" restart: "no" ports: @@ -31,7 +31,7 @@ services: #ENABLESSL: "false" #MQTTPORT: "1883" edge-service-prod-03: - image: quay.io/qiot/qiot-service + image: quay.io/qiotcovid19/edge-service:1.2.0 container_name: "edge-service-prod-03" restart: "no" ports: @@ -48,4 +48,4 @@ services: networks: default: external: - name: qiot \ No newline at end of file + name: edge \ No newline at end of file diff --git a/edge/sensors/docker-compose.yaml b/edge/sensors/docker-compose.yaml index d3d2246..56ec6cc 100644 --- a/edge/sensors/docker-compose.yaml +++ b/edge/sensors/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3' services: edge-sensors: - image: quay.io/qiot/qiot-sensors-emulator:1-x86_64 + image: quay.io/qiotcovid19/edge-sensors-emulator:1.0.1-x86_64 container_name: "edge-sensors" restart: "no" ports: @@ -9,4 +9,4 @@ services: networks: default: external: - name: qiot \ No newline at end of file + name: edge \ No newline at end of file diff --git a/mongotest/init-mongo.js b/mongotest/init-mongo.js deleted file mode 100644 index 2eeddf5..0000000 --- a/mongotest/init-mongo.js +++ /dev/null @@ -1,14 +0,0 @@ -use qiot -db.createUser( - { - user: "qiot", - pwd: "qiot", - roles: [ { role: "readWrite", db: "qiot" } ] - } -) -db.counters.insert( - { - _id: "userid", - seq: NumberInt(0) - } -) diff --git a/mongotest/start.sh b/mongotest/start.sh deleted file mode 100755 index 545012a..0000000 --- a/mongotest/start.sh +++ /dev/null @@ -1,4 +0,0 @@ -docker-compose down -sudo rm -rf volumes/mongo/* -sudo rm -rf mongo-config/* -docker-compose up \ No newline at end of file diff --git a/mongotest/stop.sh b/mongotest/stop.sh deleted file mode 100755 index 1c3ae67..0000000 --- a/mongotest/stop.sh +++ /dev/null @@ -1,2 +0,0 @@ -docker-compose down -sudo rm -rf volumes/mongo/*