From 4cf2eebd763ccdb230dd816950915511a4a93b74 Mon Sep 17 00:00:00 2001 From: abattagl Date: Fri, 22 Jan 2021 10:31:44 +0100 Subject: [PATCH 01/15] added Mongo Express, InfluxDBv2 and Grafana to the "comp" group --- comp/.gitignore | 1 - comp/docker-compose.yaml | 66 ++++++++++++------- .../provisioning/dashboards/dashboard.yml | 9 +++ .../provisioning/datasources/datasource.yml | 11 ++++ 4 files changed, 62 insertions(+), 25 deletions(-) create mode 100644 comp/volumes/grafana/provisioning/dashboards/dashboard.yml create mode 100644 comp/volumes/grafana/provisioning/datasources/datasource.yml diff --git a/comp/.gitignore b/comp/.gitignore index eaf90e9..e69de29 100644 --- a/comp/.gitignore +++ b/comp/.gitignore @@ -1 +0,0 @@ -/volumes/ diff --git a/comp/docker-compose.yaml b/comp/docker-compose.yaml index b7ae018..a82b56a 100644 --- a/comp/docker-compose.yaml +++ b/comp/docker-compose.yaml @@ -47,33 +47,51 @@ services: container_name: "mongo" environment: MONGO_INITDB_DATABASE: "qiot" - #MONGO_INITDB_ROOT_USERNAME: "root" - #MONGO_INITDB_ROOT_PASSWORD: "root" volumes: - #- ./init-mongo.js: /docker-entrypoint-initdb.d/init-mongo.js:ro - - ./volumes/mongo:/data/db + - mongodb-storage:/data/db ports: - "27017-27019:27017-27019" -# mongo-express: -# image: mongo-express -# container_name: mongo-express -# environment: -# ME_CONFIG_MONGODB_SERVER: "mongo" -# ME_CONFIG_MONGODB_PORT: "27017" -# #ME_CONFIG_MONGODB_ENABLE_ADMIN: "false" -# #ME_CONFIG_MONGODB_AUTH_DATABASE: "qiot" -# #ME_CONFIG_MONGODB_AUTH_USERNAME: "qiot" -# #ME_CONFIG_MONGODB_AUTH_PASSWORD: "qiot" -# ports: -# - "8081:8081" -# influx: -# image: "quay.io/influxdb/influxdb:v2.0.3" -# container_name: "influx" -# volumes: -# - ./volumes/influxdb2:/root/.influxdbv2 -# ports: -# - "8086:8086" -# - "9999:9999" + mongo-express: + image: "mongo-express" + container_name: "mongo-express" + environment: + ME_CONFIG_MONGODB_SERVER: "mongo" + ME_CONFIG_MONGODB_PORT: "27017" + #ME_CONFIG_MONGODB_ENABLE_ADMIN: "false" + ME_CONFIG_MONGODB_AUTH_DATABASE: "qiot" + ME_CONFIG_MONGODB_AUTH_USERNAME: "qiot" + ME_CONFIG_MONGODB_AUTH_PASSWORD: "qiot" + ports: + - "8081:8081" + influxdb: + image: "quay.io/influxdb/influxdb:v2.0.3" + container_name: "influxdb" + ports: + - "8086:8086" + - "9999:9999" + volumes: + - influxdb-storage:/root/.influxdbv2 + environment: + - INFLUXDB_DB=qiot + - INFLUXDB_ADMIN_USER=qiot + - INFLUXDB_ADMIN_PASSWORD=qiot + grafana: + image: "grafana/grafana" + container_name: "grafana" + ports: + - "3000:3000" + volumes: + - grafana-storage:/var/lib/grafana + - ./volumes/grafana/provisioning:/etc/grafana/provisioning + depends_on: + - influxdb + environment: + - GF_SECURITY_ADMIN_USER=qiot + - GF_SECURITY_ADMIN_PASSWORD=qiot +volumes: + mongodb-storage: + influxdb-storage: + grafana-storage: networks: datahub: external: true diff --git a/comp/volumes/grafana/provisioning/dashboards/dashboard.yml b/comp/volumes/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..d093eea --- /dev/null +++ b/comp/volumes/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +providers: +- name: InfluxDB + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/comp/volumes/grafana/provisioning/datasources/datasource.yml b/comp/volumes/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..6fa4bd6 --- /dev/null +++ b/comp/volumes/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,11 @@ +apiVersion: 1 +datasources: + - name: InfluxDB + type: influxdb + access: proxy + database: qiotcovid19 + user: qiot + password: qiotcovid19 + url: http://influxdb:8086 + isDefault: true + editable: true From 6e3fa748e1e6381eaf41fb8016112c97ea678e75 Mon Sep 17 00:00:00 2001 From: abattagl Date: Sat, 6 Feb 2021 10:24:50 +0100 Subject: [PATCH 02/15] Containers get always restarted Added collector services to datahub compose --- comp/docker-compose.yaml | 82 +++++++++++++------- datahub/docker-compose.yaml | 146 ++++++++++++++++++++++++------------ 2 files changed, 153 insertions(+), 75 deletions(-) diff --git a/comp/docker-compose.yaml b/comp/docker-compose.yaml index a82b56a..ee023bc 100644 --- a/comp/docker-compose.yaml +++ b/comp/docker-compose.yaml @@ -3,7 +3,7 @@ services: activemq: image: vromero/activemq-artemis:2-alpine-latest container_name: "activemq" - restart: "no" + restart: "always" networks: - datahub - qiotconnect @@ -18,7 +18,7 @@ services: # kafdrop: # image: obsidiandynamics/kafdrop # container_name: "kafkadrop" -# restart: "no" +# restart: "always" # ports: # - "9000:9000" # environment: @@ -29,12 +29,12 @@ services: kafka: image: obsidiandynamics/kafka container_name: "kafka" - restart: "no" + restart: "always" ports: - "2181:2181" - "9092:9092" environment: - KAFKA_LISTENERS: "INTERNAL://:29092,EXTERNAL://kafka:9092" + KAFKA_LISTENERS: "INTERNAL://:29092,EXTERNAL://:9092" KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka:29092,EXTERNAL://kafka:9092" KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT" KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" @@ -42,27 +42,48 @@ services: KAFKA_RESTART_ATTEMPTS: "10" KAFKA_RESTART_DELAY: "5" ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: "0" - mongo: - image: "mongo" - container_name: "mongo" - environment: - MONGO_INITDB_DATABASE: "qiot" - volumes: - - mongodb-storage:/data/db - ports: - - "27017-27019:27017-27019" - mongo-express: - image: "mongo-express" - container_name: "mongo-express" +# postgres: +# image: "postgres" # use latest official postgres version +# container_name: "postgres" +# restart: "always" +# ports: +# - "5432:5432" +# environment: +# POSTGRES_USER: "qiot" +# POSTGRES_PASSWORD: "qiot" +# POSTGRES_DB: "qiot" +# volumes: +# - postgres-storage:/var/lib/postgresql/data/ +# mongo: +# image: "mongo" +# container_name: "mongo" +# environment: +# MONGO_INITDB_DATABASE: "qiot" +# volumes: +# - mongodb-storage:/data/db +# ports: +# - "27017-27019:27017-27019" + + mysql: + image: "mysql" + container_name: "mysql" + command: --default-authentication-plugin=mysql_native_password + restart: always environment: - ME_CONFIG_MONGODB_SERVER: "mongo" - ME_CONFIG_MONGODB_PORT: "27017" - #ME_CONFIG_MONGODB_ENABLE_ADMIN: "false" - ME_CONFIG_MONGODB_AUTH_DATABASE: "qiot" - ME_CONFIG_MONGODB_AUTH_USERNAME: "qiot" - ME_CONFIG_MONGODB_AUTH_PASSWORD: "qiot" + MYSQL_ROOT_PASSWORD: "qiot" + MYSQL_DATABASE: "qiot" + MYSQL_USER: "qiot" + MYSQL_PASSWORD: "qiot" ports: - - "8081:8081" + # : < MySQL Port running inside container> + - '3306:3306' + expose: + # Opens port 3306 on the container + - '3306' + # Where our data will be persisted + volumes: + - mysql-storage:/var/lib/mysql + influxdb: image: "quay.io/influxdb/influxdb:v2.0.3" container_name: "influxdb" @@ -72,9 +93,9 @@ services: volumes: - influxdb-storage:/root/.influxdbv2 environment: - - INFLUXDB_DB=qiot - - INFLUXDB_ADMIN_USER=qiot - - INFLUXDB_ADMIN_PASSWORD=qiot + INFLUXDB_DB: "qiot" + INFLUXDB_ADMIN_USER: "qiot" + INFLUXDB_ADMIN_PASSWORD: "qiot" grafana: image: "grafana/grafana" container_name: "grafana" @@ -86,12 +107,15 @@ services: depends_on: - influxdb environment: - - GF_SECURITY_ADMIN_USER=qiot - - GF_SECURITY_ADMIN_PASSWORD=qiot + GF_SECURITY_ADMIN_USER: "qiot" + GF_SECURITY_ADMIN_PASSWORD: "qiot" + volumes: - mongodb-storage: influxdb-storage: grafana-storage: + mysql-storage: +# mongodb-storage: +# postgres-storage: networks: datahub: external: true diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index 67b0b08..ba8de13 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -1,62 +1,116 @@ version: '3' services: - datahub-registration: - image: quay.io/qiotcovid19/qiot-datahub-registration:1.0.1 - container_name: "datahub-registration" - restart: "no" + localization-service: + image: quay.io/qiotcovid19/qiot-datahub-localization:1.0.0-alpha + container_name: "localization-service" + restart: "always" networks: - datahub - - qiotconnect - ports: - - "5016:5016" - environment: - QUARKUS_PROFILE: "staging" - HTTPPORT: "5016" - MONGODBURL: "mongo:27017" - datahub-aggregation: - image: quay.io/qiotcovid19/qiot-datahub-aggregation:1.0.1 - container_name: "datahub-aggregation" - restart: "no" - ports: - - "5018:5018" + expose: + - "8080" + + station-service: + image: quay.io/qiotcovid19/qiot-datahub-station:1.0.0-alpha + container_name: "station-service" + restart: "always" + networks: + - datahub + expose: + - "8080" environment: - QUARKUS_PROFILE: "staging" - HTTPPORT: "5018" - MONGODBURL: "mongo:27017" - datahub-query: - image: quay.io/qiotcovid19/qiot-datahub-query:1.0.1 - container_name: "datahub-query" - restart: "no" + DBHOST: "mysql" + LOCALIZATION-SERVICE-URL: "http://localization-service:8080" + + registration-service: + image: quay.io/qiotcovid19/qiot-datahub-registration:2.0.0-alpha + container_name: "registration-service" + restart: "always" + networks: + - datahub + - qiotconnect + expose: + - "8080" ports: - - "5200:5200" + - "10001:8080" environment: - QUARKUS_PROFILE: "staging" - HTTPPORT: "5200" - MONGODBURL: "mongo:27017" - endpoint-mqtt: - image: quay.io/qiotcovid19/qiot-datahub-endpoint-mqtt:1.0.1 - container_name: "datahub-endpoint-mqtt" - restart: "no" + STATION-SERVICE-URL: "http://station-service:8080" + CERTLOCAL: "true" + LOGLEVEL: "FINE" + + gas-collector: + image: quay.io/qiotcovid19/qiot-datahub-collector-gas:1.0.0-alpha + container_name: "gas-collector" + restart: "always" + networks: + - datahub environment: QUARKUS_PROFILE: "staging" AMQPHOST: "activemq" MQTTHOST: "activemq" - gasstorer: - image: quay.io/qiotcovid19/qiot-datahub-storer-gas:1.0.1 - container_name: "datahub-storer-gas" - restart: "no" - environment: - QUARKUS_PROFILE: "staging" - KAFKABOOTSTRAPURL: "kafka:9092" - MONGODBURL: "mongo:27017" - pollutionstorer: - image: quay.io/qiotcovid19/qiot-datahub-storer-pollution:1.0.1 - container_name: "datahub-storer-pollution" - restart: "no" + pollution-collector: + image: quay.io/qiotcovid19/qiot-datahub-collector-pollution:1.0.0-alpha + container_name: "pollution-collector" + restart: "always" + networks: + - datahub environment: QUARKUS_PROFILE: "staging" - KAFKABOOTSTRAPURL: "kafka:9092" - MONGODBURL: "mongo:27017" + AMQPHOST: "activemq" + MQTTHOST: "activemq" + +# datahub-aggregation: +# image: quay.io/qiotcovid19/qiot-datahub-aggregation:1.0.1 +# container_name: "datahub-aggregation" +# restart: "always" +# ports: +# - "5018:5018" +# environment: +# QUARKUS_PROFILE: "staging" +# HTTPPORT: "5018" +# MONGODBURL: "mongo:27017" +# datahub-query: +# image: quay.io/qiotcovid19/qiot-datahub-query:1.0.1 +# container_name: "datahub-query" +# restart: "always" +# ports: +# - "5200:5200" +# environment: +# QUARKUS_PROFILE: "staging" +# HTTPPORT: "5200" +# MONGODBURL: "mongo:27017" +# endpoint-mqtt: +# image: quay.io/qiotcovid19/qiot-datahub-endpoint-mqtt:1.0.1 +# container_name: "datahub-endpoint-mqtt" +# restart: "always" +# environment: +# QUARKUS_PROFILE: "staging" +# AMQPHOST: "activemq" +# MQTTHOST: "activemq" + +# gasstorer: +# image: quay.io/qiotcovid19/qiot-datahub-storer-gas:1.0.1 +# container_name: "datahub-storer-gas" +# restart: "always" +# environment: +# QUARKUS_PROFILE: "staging" +# KAFKABOOTSTRAPURL: "kafka:9092" +# INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" +# INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" +# INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" +# INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" +# INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" +# pollutionstorer: +# image: quay.io/qiotcovid19/qiot-datahub-storer-pollution:1.0.1 +# container_name: "datahub-storer-pollution" +# restart: "always" +# environment: +# QUARKUS_PROFILE: "staging" +# KAFKABOOTSTRAPURL: "kafka:9092" +# INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" +# INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" +# INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" +# INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" +# INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" networks: datahub: external: true From aa2d9d79504420e23ef42dff240c168822fc973b Mon Sep 17 00:00:00 2001 From: abattagl Date: Fri, 12 Feb 2021 09:12:43 +0100 Subject: [PATCH 03/15] Commit for tested beta verion --- buildQiot.sh | 27 +++++----- comp/init-mongo.js | 15 ------ datahub/docker-compose.yaml | 78 +++++++++++++++------------- edge/application/docker-compose.yaml | 36 ++++++++----- edge/sensors/docker-compose.yaml | 2 +- 5 files changed, 79 insertions(+), 79 deletions(-) delete mode 100644 comp/init-mongo.js diff --git a/buildQiot.sh b/buildQiot.sh index 8c447c8..8e77365 100755 --- a/buildQiot.sh +++ b/buildQiot.sh @@ -1,24 +1,23 @@ -cd /home/abattagl/git/qiot/datahub/qiot-datahub-storer/qiot-datahub-storer-gas/ +cd /home/abattagl/git/qiot/datahub/qiot-datahub-localization/ chmod +x build.sh ./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-storer/qiot-datahub-storer-pollution/ -chmod +x build.sh -./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-aggregation/ -chmod +x build.sh -./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-query/ -chmod +x build.sh -./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-importer/ +cd /home/abattagl/git/qiot/datahub/qiot-datahub-station/ chmod +x build.sh ./build.sh cd /home/abattagl/git/qiot/datahub/qiot-datahub-registration/ chmod +x build.sh ./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-endpoint/qiot-datahub-endpoint-mqtt/ -chmod +x build.sh -./build.sh +cd /home/abattagl/git/qiot/datahub/qiot-datahub-collector/ +chmod +x build*.sh +./buildgas.sh +./buildpollution.sh +cd /home/abattagl/git/qiot/datahub/qiot-datahub-storer/ +chmod +x build*.sh +./buildgas.jvm.sh +./buildpollution.jvm.sh +#cd /home/abattagl/git/qiot/datahub/qiot-datahub-importer/ +#chmod +x build.sh +#./build.sh #cd /home/abattagl/git/qiot/edge/qiot-sensors-emulator/ #chmod +x build.sh #./build.sh diff --git a/comp/init-mongo.js b/comp/init-mongo.js deleted file mode 100644 index 2ed4c44..0000000 --- a/comp/init-mongo.js +++ /dev/null @@ -1,15 +0,0 @@ -#docker exec -it mongoContainer mongo -use qiot -db.createUser( - { - user: "qiot", - pwd: "qiot", - roles: [ { role: "readWrite", db: "qiot" } ] - } -) -db.counters.insert( - { - _id: "userid", - seq: NumberInt(0) - } -) diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index ba8de13..f06011a 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -1,29 +1,32 @@ version: '3' services: - localization-service: + qiot-datahub-localization-service: image: quay.io/qiotcovid19/qiot-datahub-localization:1.0.0-alpha - container_name: "localization-service" + container_name: "qiot-datahub-localization-service" restart: "always" networks: - datahub expose: - "8080" + environment: + QUARKUS_PROFILE: "staging" - station-service: + qiot-datahub-station-service: image: quay.io/qiotcovid19/qiot-datahub-station:1.0.0-alpha - container_name: "station-service" + container_name: "qiot-datahub-station-service" restart: "always" networks: - datahub expose: - "8080" environment: + QUARKUS_PROFILE: "staging" DBHOST: "mysql" - LOCALIZATION-SERVICE-URL: "http://localization-service:8080" + LOCALIZATION-SERVICE-URL: "http://qiot-datahub-localization-service:8080" - registration-service: + qiot-datahub-registration-service: image: quay.io/qiotcovid19/qiot-datahub-registration:2.0.0-alpha - container_name: "registration-service" + container_name: "qiot-datahub-registration-service" restart: "always" networks: - datahub @@ -33,28 +36,31 @@ services: ports: - "10001:8080" environment: - STATION-SERVICE-URL: "http://station-service:8080" + QUARKUS_PROFILE: "staging" + STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" CERTLOCAL: "true" LOGLEVEL: "FINE" - gas-collector: + qiot-datahub-gas-collector-service: image: quay.io/qiotcovid19/qiot-datahub-collector-gas:1.0.0-alpha - container_name: "gas-collector" + container_name: "qiot-datahub-gas-collector-service" restart: "always" networks: - datahub environment: QUARKUS_PROFILE: "staging" + STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" AMQPHOST: "activemq" MQTTHOST: "activemq" - pollution-collector: + qiot-datahub-pollution-collector-service: image: quay.io/qiotcovid19/qiot-datahub-collector-pollution:1.0.0-alpha - container_name: "pollution-collector" + container_name: "qiot-datahub-pollution-collector-service" restart: "always" networks: - datahub environment: QUARKUS_PROFILE: "staging" + STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" AMQPHOST: "activemq" MQTTHOST: "activemq" @@ -87,30 +93,30 @@ services: # AMQPHOST: "activemq" # MQTTHOST: "activemq" -# gasstorer: -# image: quay.io/qiotcovid19/qiot-datahub-storer-gas:1.0.1 -# container_name: "datahub-storer-gas" -# restart: "always" -# environment: -# QUARKUS_PROFILE: "staging" -# KAFKABOOTSTRAPURL: "kafka:9092" -# INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" -# INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" -# INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" -# INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" -# INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" -# pollutionstorer: -# image: quay.io/qiotcovid19/qiot-datahub-storer-pollution:1.0.1 -# container_name: "datahub-storer-pollution" -# restart: "always" -# environment: -# QUARKUS_PROFILE: "staging" -# KAFKABOOTSTRAPURL: "kafka:9092" -# INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" -# INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" -# INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" -# INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" -# INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" + qiot-datahub-gas-storer-service: + image: quay.io/qiotcovid19/qiot-datahub-storer-gas:2.0.0-alpha + container_name: "datahub-storer-gas" + restart: "always" + environment: + QUARKUS_PROFILE: "staging" + KAFKABOOTSTRAPURL: "kafka:9092" + INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" + INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" + INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" + INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" + INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" + qiot-datahub-pollution-storer-service: + image: quay.io/qiotcovid19/qiot-datahub-storer-pollution:2.0.0-alpha + container_name: "datahub-storer-pollution" + restart: "always" + environment: + QUARKUS_PROFILE: "staging" + KAFKABOOTSTRAPURL: "kafka:9092" + INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" + INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" + INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" + INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" + INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" networks: datahub: external: true diff --git a/edge/application/docker-compose.yaml b/edge/application/docker-compose.yaml index 61f5683..6194767 100644 --- a/edge/application/docker-compose.yaml +++ b/edge/application/docker-compose.yaml @@ -1,62 +1,72 @@ version: '3' services: - edge-service-01: - image: quay.io/qiotcovid19/edge-service:1.2.0-x86_64 - container_name: "edge-service-01" + qiot-edge-service-01: + image: quay.io/qiotcovid19/qiot-edge-service:2.0.0-alpha-x86_64 + container_name: "qiot-edge-service-01" restart: "no" networks: - edge - qiotconnect ports: - "8091:8080" + volumes: + - device1-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" STATIONSERIAL: "teststationserial01" STATIONNAME: "teststation01" STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" - SENSORSURL: "http://edge-sensors:5000/" + SENSORSURL: "http://qiot-edge-sensors:5000/" DATAHUBURL: "http://datahub-registration:5016/" MQTTURL: "activemq" ENABLESSL: "false" MQTTPORT: "1883" - edge-service-02: - image: quay.io/qiotcovid19/edge-service:1.2.0-x86_64 - container_name: "edge-service-02" + qiot-edge-service-02: + image: quay.io/qiotcovid19/qiot-edge-service:2.0.0-alpha-x86_64 + container_name: "qiot-edge-service-02" restart: "no" networks: - edge - qiotconnect ports: - "8092:8080" + volumes: + - device2-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" STATIONSERIAL: "teststationserial02" STATIONNAME: "teststation02" STATIONADDRESS: "soriano calabro" - SENSORSURL: "http://edge-sensors:5000/" + SENSORSURL: "http://qiot-edge-sensors:5000/" DATAHUBURL: "http://datahub-registration:5016/" MQTTURL: "activemq" ENABLESSL: "false" MQTTPORT: "1883" - edge-service-03: - image: quay.io/qiotcovid19/edge-service:1.2.0-x86_64 - container_name: "edge-service-03" + qiot-edge-service-03: + image: quay.io/qiotcovid19/qiot-edge-service:2.0.0-alpha-x86_64 + container_name: "qiot-edge-service-03" restart: "no" networks: - edge - qiotconnect ports: - "8093:8080" + volumes: + - device3-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" STATIONSERIAL: "teststationserial03" STATIONNAME: "teststation03" STATIONADDRESS: "minsk" - SENSORSURL: "http://edge-sensors:5000/" - DATAHUBURL: "http://datahub-registration:5016/" + SENSORSURL: "http://qiot-edge-sensors:5000/" + DATAHUBURL: "http://qiot-datahub-registration-service:5016/" MQTTURL: "activemq" ENABLESSL: "false" MQTTPORT: "1883" +volumes: + device1-storage: + device2-storage: + device3-storage: networks: edge: external: true diff --git a/edge/sensors/docker-compose.yaml b/edge/sensors/docker-compose.yaml index 56ec6cc..0bd987a 100644 --- a/edge/sensors/docker-compose.yaml +++ b/edge/sensors/docker-compose.yaml @@ -1,6 +1,6 @@ version: '3' services: - edge-sensors: + qiot-edge-sensors: image: quay.io/qiotcovid19/edge-sensors-emulator:1.0.1-x86_64 container_name: "edge-sensors" restart: "no" From eca26514cfe60d439cb5769afd34221fd8a47d62 Mon Sep 17 00:00:00 2001 From: abattagl Date: Fri, 12 Feb 2021 16:04:18 +0100 Subject: [PATCH 04/15] bugfixes --- datahub/docker-compose.yaml | 33 ++-------------------------- edge/application/docker-compose.yaml | 24 ++++++++++---------- edge/sensors/docker-compose.yaml | 2 +- 3 files changed, 15 insertions(+), 44 deletions(-) diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index f06011a..6668dbc 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -64,38 +64,9 @@ services: AMQPHOST: "activemq" MQTTHOST: "activemq" -# datahub-aggregation: -# image: quay.io/qiotcovid19/qiot-datahub-aggregation:1.0.1 -# container_name: "datahub-aggregation" -# restart: "always" -# ports: -# - "5018:5018" -# environment: -# QUARKUS_PROFILE: "staging" -# HTTPPORT: "5018" -# MONGODBURL: "mongo:27017" -# datahub-query: -# image: quay.io/qiotcovid19/qiot-datahub-query:1.0.1 -# container_name: "datahub-query" -# restart: "always" -# ports: -# - "5200:5200" -# environment: -# QUARKUS_PROFILE: "staging" -# HTTPPORT: "5200" -# MONGODBURL: "mongo:27017" -# endpoint-mqtt: -# image: quay.io/qiotcovid19/qiot-datahub-endpoint-mqtt:1.0.1 -# container_name: "datahub-endpoint-mqtt" -# restart: "always" -# environment: -# QUARKUS_PROFILE: "staging" -# AMQPHOST: "activemq" -# MQTTHOST: "activemq" - qiot-datahub-gas-storer-service: image: quay.io/qiotcovid19/qiot-datahub-storer-gas:2.0.0-alpha - container_name: "datahub-storer-gas" + container_name: "qiot-datahub-gas-storer-service" restart: "always" environment: QUARKUS_PROFILE: "staging" @@ -107,7 +78,7 @@ services: INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" qiot-datahub-pollution-storer-service: image: quay.io/qiotcovid19/qiot-datahub-storer-pollution:2.0.0-alpha - container_name: "datahub-storer-pollution" + container_name: "qiot-datahub-pollution-storer-service" restart: "always" environment: QUARKUS_PROFILE: "staging" diff --git a/edge/application/docker-compose.yaml b/edge/application/docker-compose.yaml index 6194767..19908bc 100644 --- a/edge/application/docker-compose.yaml +++ b/edge/application/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3' services: qiot-edge-service-01: - image: quay.io/qiotcovid19/qiot-edge-service:2.0.0-alpha-x86_64 + image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 container_name: "qiot-edge-service-01" restart: "no" networks: @@ -13,16 +13,16 @@ services: - device1-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" - STATIONSERIAL: "teststationserial01" - STATIONNAME: "teststation01" + STATIONSERIAL: "teststationserialS01" + STATIONNAME: "teststationS01" STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" SENSORSURL: "http://qiot-edge-sensors:5000/" - DATAHUBURL: "http://datahub-registration:5016/" + REGISTRATION-SERVICE-URL: "http://qiot-datahub-registration-service:8080/" MQTTURL: "activemq" ENABLESSL: "false" MQTTPORT: "1883" qiot-edge-service-02: - image: quay.io/qiotcovid19/qiot-edge-service:2.0.0-alpha-x86_64 + image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 container_name: "qiot-edge-service-02" restart: "no" networks: @@ -34,16 +34,16 @@ services: - device2-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" - STATIONSERIAL: "teststationserial02" - STATIONNAME: "teststation02" + STATIONSERIAL: "teststationserialS02" + STATIONNAME: "teststationS02" STATIONADDRESS: "soriano calabro" SENSORSURL: "http://qiot-edge-sensors:5000/" - DATAHUBURL: "http://datahub-registration:5016/" + REGISTRATION-SERVICE-URL: "http://qiot-datahub-registration-service:8080/" MQTTURL: "activemq" ENABLESSL: "false" MQTTPORT: "1883" qiot-edge-service-03: - image: quay.io/qiotcovid19/qiot-edge-service:2.0.0-alpha-x86_64 + image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 container_name: "qiot-edge-service-03" restart: "no" networks: @@ -55,11 +55,11 @@ services: - device3-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" - STATIONSERIAL: "teststationserial03" - STATIONNAME: "teststation03" + STATIONSERIAL: "teststationserialS03" + STATIONNAME: "teststationS03" STATIONADDRESS: "minsk" SENSORSURL: "http://qiot-edge-sensors:5000/" - DATAHUBURL: "http://qiot-datahub-registration-service:5016/" + REGISTRATION-SERVICE-URL: "http://qiot-datahub-registration-service:8080/" MQTTURL: "activemq" ENABLESSL: "false" MQTTPORT: "1883" diff --git a/edge/sensors/docker-compose.yaml b/edge/sensors/docker-compose.yaml index 0bd987a..47fa464 100644 --- a/edge/sensors/docker-compose.yaml +++ b/edge/sensors/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3' services: qiot-edge-sensors: image: quay.io/qiotcovid19/edge-sensors-emulator:1.0.1-x86_64 - container_name: "edge-sensors" + container_name: "qiot-edge-sensors" restart: "no" ports: - "5000:5000" From ffbd6999ac5b05cc5937f4885ede374c2f9cf994 Mon Sep 17 00:00:00 2001 From: abattagl Date: Fri, 12 Feb 2021 16:43:03 +0100 Subject: [PATCH 05/15] cleanup and readme --- README.md | 60 ++++++++++++++++++++++++++++++++++++---- comp/docker-compose.yaml | 33 +--------------------- comp/mysql-setup.sql | 15 ++++++++++ 3 files changed, 71 insertions(+), 37 deletions(-) create mode 100644 comp/mysql-setup.sql diff --git a/README.md b/README.md index 8e29d50..bcedf5b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ## qiot-datahub-docker A set od docker-compose files to run the QIoT environment on your host machine using a OCI-compliant container engine and tools. -Podman engine is recommendedthus you'll find below the prerequisited and mandatory actions to take before using these compose files: +Podman engine is recommended, thus you'll find below the prerequisites and mandatory actions to take before using these compose files: # if you want to go with podman: @@ -9,11 +9,10 @@ Podman engine is recommendedthus you'll find below the prerequisited and mandato # Set up a dedicated container network for both your edge pods and datahub pods - `$ sudo docker network create edge` - Network dedicated to the services running on the edge device /etc/cni/net.d/qiot-covid19-edge.conflist -`$ sudo podman network create datahub` - Network dedicated to the services running on the datacenter +`$ sudo docker network create datahub` - Network dedicated to the services running on the datacenter /etc/cni/net.d/qiot-covid19-datahub.conflist `$ sudo docker network create qiotconnect` - Network dedicated to the interaction between the edge device and the datahub @@ -33,7 +32,7 @@ Podman: Podman Cockpit extension We can edit the `/etc/selinux/config` text file with our persistent setting, either enforcing, permissive, or disabled. By default this file appears as shown below. -``` +```properties This file controls the state of SELinux on the system. SELINUX= can take one of these three values: enforcing - SELinux security policy is enforced. @@ -47,4 +46,55 @@ ELINUXTYPE= can take one of three two values: SELINUXTYPE=targeted ``` -We can simply edit the SELINUX variable between enforcing, permissive, or disabled, as outlined in the comments of the file. After editing the file the changes will not be in place immediately and will only change after system reboot. \ No newline at end of file +We can simply edit the SELINUX variable between enforcing, permissive, or disabled, as outlined in the comments of the file. After editing the file the changes will not be in place immediately and will only change after system reboot. + +# The QIoT-covid19 system + +The QIoT covid19 use-case implementation is made up of 3 main layers: + + - **comp**: the software infrastructure, composite of integration products to make micro-services interact with each other and to manage endpoints and data + - **datahub**: micro-services implementing the use-case business logic + - **edge**: software running at the edge device + - **sensors**: a micro-service emulating the behavior of the sensor board connected to the raspberry pi + - **edge**: the micro-service implementing the edge device business logic + +## Spinning up the environment + +Just enter each and every folder and run a docker compose or, alternatively, use Portainer.io to manage your stacks. + +Please follow the order below to make sure all the components are capable to find their dependencies accordingly: + +### software architecture + +```bash + $ cd comp + $ docker-compose up --remove-orphans +``` + +​ access the mysql instance and execute the script contained in the file *mysql-setup.sql*: + + +```bash + $ docker exec -it mysql bash -l +``` + +### business logic + +```bash + $ cd datahub + $ docker-compose up --remove-orphans +``` + +### edge device - sensors + +```bash + $ cd edge/sensors + $ docker-compose up --remove-orphans +``` + +### edge device - business logic + +```bash + $ cd edge/application + $ docker-compose up --remove-orphans +``` \ No newline at end of file diff --git a/comp/docker-compose.yaml b/comp/docker-compose.yaml index ee023bc..2947fd4 100644 --- a/comp/docker-compose.yaml +++ b/comp/docker-compose.yaml @@ -15,17 +15,7 @@ services: environment: ARTEMIS_USERNAME: "quarkus" ARTEMIS_PASSWORD: "quarkus" -# kafdrop: -# image: obsidiandynamics/kafdrop -# container_name: "kafkadrop" -# restart: "always" -# ports: -# - "9000:9000" -# environment: -# KAFKA_BROKERCONNECT: "kafka:29092" -# JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify" -# depends_on: -# - "kafka" + kafka: image: obsidiandynamics/kafka container_name: "kafka" @@ -42,27 +32,6 @@ services: KAFKA_RESTART_ATTEMPTS: "10" KAFKA_RESTART_DELAY: "5" ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: "0" -# postgres: -# image: "postgres" # use latest official postgres version -# container_name: "postgres" -# restart: "always" -# ports: -# - "5432:5432" -# environment: -# POSTGRES_USER: "qiot" -# POSTGRES_PASSWORD: "qiot" -# POSTGRES_DB: "qiot" -# volumes: -# - postgres-storage:/var/lib/postgresql/data/ -# mongo: -# image: "mongo" -# container_name: "mongo" -# environment: -# MONGO_INITDB_DATABASE: "qiot" -# volumes: -# - mongodb-storage:/data/db -# ports: -# - "27017-27019:27017-27019" mysql: image: "mysql" diff --git a/comp/mysql-setup.sql b/comp/mysql-setup.sql new file mode 100644 index 0000000..f84987e --- /dev/null +++ b/comp/mysql-setup.sql @@ -0,0 +1,15 @@ +CREATE TABLE `qiot`.`station` ( + `id` binary(16) NOT NULL, + `city` varchar(255) DEFAULT NULL, + `country` varchar(255) DEFAULT NULL, + `country_code` varchar(255) DEFAULT NULL, + `latitude` double NOT NULL, + `longitude` double NOT NULL, + `name` varchar(255) NOT NULL, + `registered_on` timestamp NULL DEFAULT NULL, + `serial` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `UK_n8mucdp0jk0ykhjyikkjifjnp` (`serial`), + KEY `location_idx` (`country_code`,`city`), + KEY `voordinatex_idx` (`longitude`,`latitude`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; \ No newline at end of file From 752538175972caa439366cbcdf5f0e040c3624e1 Mon Sep 17 00:00:00 2001 From: abattagl Date: Thu, 18 Feb 2021 17:49:17 +0100 Subject: [PATCH 06/15] openshift scripts; cleanup;edge-service prod --- datahub/docker-compose.yaml | 1 + edge-server/application/docker-compose.yaml | 35 ---- edge-server/sensors/docker-compose.yaml | 39 ----- .../sensors/volumes/rabbitmq/.gitignore | 1 - .../sensors/volumes/rabbitmq/enable_plugins | 1 - edge/prod/docker-compose.yaml | 78 +++++---- influxdbtest/docker-compose.yaml | 36 ----- openshift.txt | 149 +++++++++++++----- 8 files changed, 156 insertions(+), 184 deletions(-) delete mode 100644 edge-server/application/docker-compose.yaml delete mode 100644 edge-server/sensors/docker-compose.yaml delete mode 100644 edge-server/sensors/volumes/rabbitmq/.gitignore delete mode 100644 edge-server/sensors/volumes/rabbitmq/enable_plugins delete mode 100644 influxdbtest/docker-compose.yaml diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index 6668dbc..aa2a0b2 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -10,6 +10,7 @@ services: - "8080" environment: QUARKUS_PROFILE: "staging" + LOGLEVEL: "FINE" qiot-datahub-station-service: image: quay.io/qiotcovid19/qiot-datahub-station:1.0.0-alpha diff --git a/edge-server/application/docker-compose.yaml b/edge-server/application/docker-compose.yaml deleted file mode 100644 index cd90211..0000000 --- a/edge-server/application/docker-compose.yaml +++ /dev/null @@ -1,35 +0,0 @@ -version: '3' -services: - edge-integrator: - image: quay.io/qiotcovid19/qiot-edge-integrator:1-x86_64 - container_name: "edge-integrator" - restart: "no" - environment: - AMQPHOST: "edge-messaging" - AMQPPORT: 5672 - AMQPUSERNAME: "qiot" - AMQPPASSWORD: "qiot" - SENSORSURL: "http://edge-sensors:5000/" - edge-server-01: - image: quay.io/qiotcovid19/qiot-edge-server:1-x86_64 - container_name: "edge-server-01" - restart: "no" - ports: - - "8091:8080" - environment: - AMQPHOST: "edge-messaging" - AMQPPORT: 5672 - AMQPUSERNAME: "qiot" - AMQPPASSWORD: "qiot" - STATIONSERIAL: "teststationserial01" - STATIONNAME: "teststation01" - STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" - SENSORSURL: "http://edge-sensors:5000/" - DATAHUBURL: "http://datahub-registration:5016/" - MQTTURL: "activemq" - ENABLESSL: "false" - MQTTPORT: "1883" -networks: - default: - external: - name: edge \ No newline at end of file diff --git a/edge-server/sensors/docker-compose.yaml b/edge-server/sensors/docker-compose.yaml deleted file mode 100644 index 6d187dc..0000000 --- a/edge-server/sensors/docker-compose.yaml +++ /dev/null @@ -1,39 +0,0 @@ -version: '3' -services: - edge-messaging: - image: rabbitmq:3 - container_name: edge-messaging - volumes: -# - ./volumes/rabbitmq/etc/:/etc/rabbitmq/ - - ./volumes/rabbitmq/data/:/var/lib/rabbitmq/ - - ./volumes/rabbitmq/logs/:/var/log/rabbitmq/ - - ./volumes/rabbitmq/enable_plugins:/etc/rabbitmq/enabled_plugins - environment: -# RABBITMQ_ERLANG_COOKIE: ${RABBITMQ_ERLANG_COOKIE} - RABBITMQ_DEFAULT_USER: qiot - RABBITMQ_DEFAULT_PASS: qiot - ports: - - 5673:5672 - - 15672:15672 - expose: - - "5672" - edge-sensors: - image: quay.io/qiotcovid19/qiot-sensors-emulator - container_name: "edge-sensors" - restart: "no" - ports: - - "5000:5000" - environment: - QUARKUS_PROFILE: "staging" - edge-sensors: - image: quay.io/qiotcovid19/qiot-sensors-emulator - container_name: "edge-sensors" - restart: "no" - ports: - - "5000:5000" - environment: - QUARKUS_PROFILE: "staging" -networks: - default: - external: - name: edge \ No newline at end of file diff --git a/edge-server/sensors/volumes/rabbitmq/.gitignore b/edge-server/sensors/volumes/rabbitmq/.gitignore deleted file mode 100644 index 82f0c3a..0000000 --- a/edge-server/sensors/volumes/rabbitmq/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/data/ diff --git a/edge-server/sensors/volumes/rabbitmq/enable_plugins b/edge-server/sensors/volumes/rabbitmq/enable_plugins deleted file mode 100644 index c5f2ce4..0000000 --- a/edge-server/sensors/volumes/rabbitmq/enable_plugins +++ /dev/null @@ -1 +0,0 @@ -[rabbitmq_amqp1_0,rabbitmq_management,rabbitmq_web_dispatch,rabbitmq_management_agent]. diff --git a/edge/prod/docker-compose.yaml b/edge/prod/docker-compose.yaml index e04f8df..32b697f 100644 --- a/edge/prod/docker-compose.yaml +++ b/edge/prod/docker-compose.yaml @@ -1,51 +1,63 @@ version: '3' services: - edge-service-prod-01: - image: quay.io/qiotcovid19/edge-service:1.2.0 - container_name: "edge-service-prod-01" + qiot-edge-service-P-01: + image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 + container_name: "qiot-edge-service-P-01" restart: "no" + networks: + - edge ports: - "8091:8080" + volumes: + - qiot-edge-service-P-01-storage:/var/data/qiot/ environment: - STATIONSERIAL: "teststationserial01" - STATIONNAME: "teststation01" + STATIONSERIAL: "test-station-serialP01" + STATIONNAME: "test-station-P01" STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" - SENSORSURL: "http://edge-sensors:5000/" - #DATAHUBURL: "http://datahub-registration:5016/" - #MQTTHOST: "activemq" - #ENABLESSL: "false" - #MQTTPORT: "1883" - edge-service-prod-02: - image: quay.io/qiotcovid19/edge-service:1.2.0 - container_name: "edge-service-prod-02" + SENSORSURL: "http://qiot-edge-sensors:5000/" + REGISTRATION-SERVICE-URL: "http://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" + MQTTPORT: "443" + qiot-edge-service-P-02: + image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 + container_name: "qiot-edge-service-P-02" restart: "no" + networks: + - edge ports: - "8092:8080" + volumes: + - qiot-edge-service-P-02-storage:/var/data/qiot/ environment: - STATIONSERIAL: "teststationserial02" - STATIONNAME: "teststation02" + STATIONSERIAL: "test-station-serialP02" + STATIONNAME: "test-station-P02" STATIONADDRESS: "soriano calabro" - SENSORSURL: "http://edge-sensors:5000/" - #DATAHUBURL: "http://datahub-registration:5016/" - #MQTTHOST: "activemq" - #ENABLESSL: "false" - #MQTTPORT: "1883" - edge-service-prod-03: - image: quay.io/qiotcovid19/edge-service:1.2.0 - container_name: "edge-service-prod-03" + SENSORSURL: "http://qiot-edge-sensors:5000/" + REGISTRATION-SERVICE-URL: "http://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" + MQTTPORT: "443" + qiot-edge-service-P-03: + image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 + container_name: "qiot-edge-service-P-03" restart: "no" + networks: + - edge ports: - "8093:8080" + volumes: + - qiot-edge-service-P-03-storage:/var/data/qiot/ environment: - STATIONSERIAL: "teststationserial03" - STATIONNAME: "teststation03" + STATIONSERIAL: "test-station-serialP03" + STATIONNAME: "test-station-P03" STATIONADDRESS: "minsk" - SENSORSURL: "http://edge-sensors:5000/" - #DATAHUBURL: "http://datahub-registration:5016/" - #MQTTHOST: "activemq" - #ENABLESSL: "false" - #MQTTPORT: "1883" + SENSORSURL: "http://qiot-edge-sensors:5000/" + REGISTRATION-SERVICE-URL: "http://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" + MQTTPORT: "443" +volumes: + qiot-edge-service-P-01-storage: + qiot-edge-service-P-02-storage: + qiot-edge-service-P-03-storage: networks: - default: - external: - name: edge \ No newline at end of file + edge: + external: true diff --git a/influxdbtest/docker-compose.yaml b/influxdbtest/docker-compose.yaml deleted file mode 100644 index 6c6179f..0000000 --- a/influxdbtest/docker-compose.yaml +++ /dev/null @@ -1,36 +0,0 @@ -version: '3' -services: - influxdb: - image: influxdb:1.8.2 - container_name: "influxdb" - ports: - - "9999:9999" - - "8086:8086" - volumes: - - ./volumes/influxdb:/data/db - - chronograph: - image: quay.io/influxdb/chronograf:1.8.6 - container_name: "chronograph" - entrypoint: 'chronograf --kapacitor-url=http://kapacitor:9092 --influxdb-url=http://influxdb:8086' - depends_on: - - "influxdb" - - "kapacitor" - ports: - - "8888:8888" - volumes: - - ./volumes/influxdb:/var/lib/chronograf - - kapacitor: - image: kapacitor - container_name: "kapacitor" - depends_on: - - "influxdb" - environment: - - KAPACITOR_INFLUXDB_0_URLS_0=http://influxdb:8086 - - KAPACITOR_HOSTNAME=kapacitor -networks: - default: - external: - name: influxdb - #9340172063e48e0f5473ff47b5b05403c03d2f43c0b107d710dd3e014b966b06 \ No newline at end of file diff --git a/openshift.txt b/openshift.txt index b7e70cc..0bfe1a2 100644 --- a/openshift.txt +++ b/openshift.txt @@ -1,45 +1,116 @@ +/*** LOCALIZATION ***/ +# OK +oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-localization.git --name=localization-service +# OK +oc patch bc/localization-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' +# OK +oc start-build localization-service +# OK +oc new-app localization-service + +/*** STATION ***/ +# OK +oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-station.git --name=station-service +# OK +oc patch bc/station-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' +# OK +oc start-build station-service +# OK +oc new-app station-service -e DBHOST=mysql -e LOCALIZATION-SERVICE-URL=http://localization-service:8080 + /*** REGISTRATION ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-registration.git --name=qiot-registration -oc patch bc/qiot-registration -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-registration -oc new-app qiot-registration -e MONGODBURL=mongodb:27017 -oc expose svc/qiot-registration +# OK +oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-registration.git#v2 --name=registration-service +# OK +oc patch bc/registration-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' +# OK +oc start-build registration-service +# OK +oc new-app registration-service -e STATION-SERVICE-URL=http://station-service:8080 -e CERTLOCAL=true +# OK +oc expose svc/registration-service -/*** IMPORTER ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-importer.git --name=qiot-importer -oc patch bc/qiot-importer -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-importer -oc new-app qiot-importer -e MONGODBURL=mongodb:27017 -oc expose svc/qiot-importer - -/*** MQTT ENDPOINT ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-endpoint.git --context-dir=qiot-datahub-endpoint-mqtt --name=qiot-endpoint-mqtt -oc patch bc/qiot-endpoint-mqtt -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-endpoint-mqtt -oc new-app qiot-endpoint-mqtt -e MQTTHOST=qiot-broker-hdls-svc -e KAFKABOOTSTRAPURL=kafka-cluster-kafka-bootstrap:9092 +/*** GAS COLLECTOR ***/ +# OK +oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-collector.git \ + --context-dir=. --name=gas-collector-service --build-env='ARTIFACT_DIR=qiot-datahub-collector-gas/target' \ + --build-env='MVN_ADDITIONAL_ARGS= -pl qiot-datahub-collector-gas -am' +# OK +oc patch bc/gas-collector-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' +# OK +oc start-build gas-collector-service +# TODO +oc new-app gas-collector-service \ +-e MQTTHOST=endpoint-service-hdls-svc \ +-e STATION-SERVICE-URL=http://station-service:8080 \ +-e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 + +/*** POLLUTION COLLECTOR ***/ +# OK +oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-collector.git \ + --context-dir=. --name=pollution-collector-service --build-env='ARTIFACT_DIR=qiot-datahub-collector-pollution/target' \ + --build-env='MVN_ADDITIONAL_ARGS= -pl qiot-datahub-collector-pollution -am ' +# OK +oc patch bc/pollution-collector-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' +# OK +oc start-build pollution-collector-service +# TODO +oc new-app pollution-collector-service \ +-e MQTTHOST=endpoint-service-hdls-svc \ +-e STATION-SERVICE-URL=http://station-service:8080 \ +-e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 /*** GAS STORER ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-storer.git --context-dir=qiot-datahub-storer-gas --name=qiot-storer-gas -oc patch bc/qiot-storer-gas -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-storer-gas -oc new-app qiot-storer-gas -e MONGODBURL=mongodb:27017 -e KAFKABOOTSTRAPURL=kafka-cluster-kafka-bootstrap:9092 +# OK +oc new-build registry.access.redhat.com/openjdk/openjdk-11-rhel7~https://github.com/qiot-project/qiot-datahub-storer.git#v2 \ + --context-dir=. --name=gas-storer-service \ + --build-env='MAVEN_ARGS= -pl qiot-datahub-storer-gas -am' \ + --build-env='MAVEN_S2I_ARTIFACT_DIRS=qiot-datahub-storer-gas/target' \ + --build-env='S2I_SOURCE_DEPLOYMENTS_FILTER=*-runner.jar lib' +# OK +oc new-app gas-storer-service \ + -e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 \ + -e INFLUXDB_CONNECTION_STRING=http://timeseries-service-influxdb2 \ + -e INFLUXDB_ORG_ID=qiot \ # 2d67bc8b65cc1c19 + -e INFLUXDB_DATA_BUCKET_ID=covid19 \ # 507dfe460a44c268 + -e INFLUXDB_DATA_BUCKET_NAME=covid19 \ + -e INFLUXDB_TOKEN=JCNTRpJj6AsvqiD7MnRWPZhIrXYpHwusT_Mwvc0I26E6lFJsXNzkaD2SbkIxeeH3TbiakNbbZGm_EKk8dlCCbQ== /*** POLLUTION STORER ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-storer.git --context-dir=qiot-datahub-storer-pollution --name=qiot-storer-pollution -oc patch bc/qiot-storer-pollution -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-storer-pollution -oc new-app qiot-storer-pollution -e MONGODBURL=mongodb:27017 -e KAFKABOOTSTRAPURL=kafka-cluster-kafka-bootstrap:9092 - -/*** AGGREGATION ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-aggregation.git --name=qiot-aggregation -oc patch bc/qiot-aggregation -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-aggregation -oc new-app qiot-aggregation -e MONGODBURL=mongodb:27017 -oc expose svc/qiot-aggregation - -/*** QUERY ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-query.git --name=qiot-query -oc patch bc/qiot-query -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-query -oc new-app qiot-query -e MONGODBURL=mongodb:27017 -oc expose svc/qiot-query \ No newline at end of file +# OK +oc new-build registry.access.redhat.com/openjdk/openjdk-11-rhel7~https://github.com/qiot-project/qiot-datahub-storer.git#v2 \ + --context-dir=. --name=pollution-storer-service \ + --build-env='MAVEN_ARGS= -pl qiot-datahub-storer-pollution -am' \ + --build-env='MAVEN_S2I_ARTIFACT_DIRS=qiot-datahub-storer-pollution/target' \ + --build-env='S2I_SOURCE_DEPLOYMENTS_FILTER=*-runner.jar lib' +# OK +oc new-app pollution-storer-service \ + -e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 \ + -e INFLUXDB_CONNECTION_STRING=http://timeseries-service-influxdb2 \ + -e INFLUXDB_ORG_ID=qiot \ + -e INFLUXDB_DATA_BUCKET_ID=covid19 \ + -e INFLUXDB_DATA_BUCKET_NAME=covid19 \ + -e INFLUXDB_TOKEN=JCNTRpJj6AsvqiD7MnRWPZhIrXYpHwusT_Mwvc0I26E6lFJsXNzkaD2SbkIxeeH3TbiakNbbZGm_EKk8dlCCbQ== + +/*** IMPORTER ***/ +# OK +oc new-build registry.access.redhat.com/ubi8/openjdk-11~https://github.com/qiot-project/qiot-datahub-importer.git#v2 --name=importer-service +# TODO +oc new-app importer-service -e MONGODBURL=mongodb:27017 +# OK +oc expose svc/importer-service + + + + + + + + + + + + + + + From 79d3836a2e0c58f1912bd2ad95107336a2d0c404 Mon Sep 17 00:00:00 2001 From: Andrea Battaglia Date: Tue, 2 Mar 2021 10:45:31 +0100 Subject: [PATCH 07/15] switched to postgresql --- buildQiot.sh | 20 +++++----- comp/docker-compose.yaml | 56 +++++++++++++++++++--------- comp/mysql-setup.sql | 15 -------- datahub/docker-compose.yaml | 19 +++++++++- edge/application/docker-compose.yaml | 6 +-- 5 files changed, 70 insertions(+), 46 deletions(-) delete mode 100644 comp/mysql-setup.sql diff --git a/buildQiot.sh b/buildQiot.sh index 8e77365..37fa83a 100755 --- a/buildQiot.sh +++ b/buildQiot.sh @@ -1,27 +1,27 @@ -cd /home/abattagl/git/qiot/datahub/qiot-datahub-localization/ +cd ~/git/qiot/datahub/qiot-datahub-localization/ chmod +x build.sh ./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-station/ +cd ~/git/qiot/datahub/qiot-datahub-station/ chmod +x build.sh ./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-registration/ +cd ~/git/qiot/datahub/qiot-datahub-registration/ chmod +x build.sh ./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-collector/ +cd ~/git/qiot/datahub/qiot-datahub-collector/ chmod +x build*.sh ./buildgas.sh ./buildpollution.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-storer/ +cd ~/git/qiot/datahub/qiot-datahub-storer/ chmod +x build*.sh ./buildgas.jvm.sh ./buildpollution.jvm.sh -#cd /home/abattagl/git/qiot/datahub/qiot-datahub-importer/ +cd ~/git/qiot/datahub/qiot-datahub-importer/ +chmod +x build.jvm.sh +./build.jvm.sh +#cd ~/git/qiot/edge/qiot-sensors-emulator/ #chmod +x build.sh #./build.sh -#cd /home/abattagl/git/qiot/edge/qiot-sensors-emulator/ -#chmod +x build.sh -#./build.sh -#cd /home/abattagl/git/qiot/edge/qiot-service/ +#cd ~/git/qiot/edge/qiot-service/ #chmod +x build.sh #./build.sh diff --git a/comp/docker-compose.yaml b/comp/docker-compose.yaml index 2947fd4..da40698 100644 --- a/comp/docker-compose.yaml +++ b/comp/docker-compose.yaml @@ -33,25 +33,47 @@ services: KAFKA_RESTART_DELAY: "5" ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: "0" - mysql: - image: "mysql" - container_name: "mysql" - command: --default-authentication-plugin=mysql_native_password +# mysql: +# image: "mysql" +# container_name: "mysql" +# command: --default-authentication-plugin=mysql_native_password +# restart: always +# environment: +# MYSQL_ROOT_PASSWORD: "qiot" +# MYSQL_DATABASE: "qiot" +# MYSQL_USER: "qiot" +# MYSQL_PASSWORD: "qiot" +# ports: +# # : < MySQL Port running inside container> +# - '3306:3306' +# expose: +# # Opens port 3306 on the container +# - '3306' +# # Where our data will be persisted +# volumes: +# - mysql-storage:/var/lib/mysql +# security_opt: +# - seccomp:unconfined + + postgresql: + image: postgres + container_name: "postgresql" restart: always - environment: - MYSQL_ROOT_PASSWORD: "qiot" - MYSQL_DATABASE: "qiot" - MYSQL_USER: "qiot" - MYSQL_PASSWORD: "qiot" ports: - # : < MySQL Port running inside container> - - '3306:3306' - expose: - # Opens port 3306 on the container - - '3306' - # Where our data will be persisted + - "5432:5432" volumes: - - mysql-storage:/var/lib/mysql + - postgres-storage:/var/lib/postgresql/data + environment: + POSTGRES_USER: "qiot" + POSTGRES_PASSWORD: "qiot" + POSTGRES_DB: "qiot" + + postgresqladminer: + image: adminer + container_name: "postgresqladminer" + restart: always + ports: + - "18080:8080" influxdb: image: "quay.io/influxdb/influxdb:v2.0.3" @@ -84,7 +106,7 @@ volumes: grafana-storage: mysql-storage: # mongodb-storage: -# postgres-storage: + postgres-storage: networks: datahub: external: true diff --git a/comp/mysql-setup.sql b/comp/mysql-setup.sql deleted file mode 100644 index f84987e..0000000 --- a/comp/mysql-setup.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE TABLE `qiot`.`station` ( - `id` binary(16) NOT NULL, - `city` varchar(255) DEFAULT NULL, - `country` varchar(255) DEFAULT NULL, - `country_code` varchar(255) DEFAULT NULL, - `latitude` double NOT NULL, - `longitude` double NOT NULL, - `name` varchar(255) NOT NULL, - `registered_on` timestamp NULL DEFAULT NULL, - `serial` varchar(255) NOT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `UK_n8mucdp0jk0ykhjyikkjifjnp` (`serial`), - KEY `location_idx` (`country_code`,`city`), - KEY `voordinatex_idx` (`longitude`,`latitude`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; \ No newline at end of file diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index aa2a0b2..36c4f0a 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -1,5 +1,22 @@ version: '3' services: + qiot-datahub-import-service: + image: quay.io/qiotcovid19/qiot-datahub-importer:2.0.0-beta + container_name: "qiot-datahub-import-service" + restart: "always" + networks: + - datahub + ports: + - "5100:8080" + environment: + QUARKUS_PROFILE: "staging" + DBHOST: "postgresql" + INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" + INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" + INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" + INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" + INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" + qiot-datahub-localization-service: image: quay.io/qiotcovid19/qiot-datahub-localization:1.0.0-alpha container_name: "qiot-datahub-localization-service" @@ -22,7 +39,7 @@ services: - "8080" environment: QUARKUS_PROFILE: "staging" - DBHOST: "mysql" + DBHOST: "postgresql" LOCALIZATION-SERVICE-URL: "http://qiot-datahub-localization-service:8080" qiot-datahub-registration-service: diff --git a/edge/application/docker-compose.yaml b/edge/application/docker-compose.yaml index 19908bc..311fba6 100644 --- a/edge/application/docker-compose.yaml +++ b/edge/application/docker-compose.yaml @@ -8,7 +8,7 @@ services: - edge - qiotconnect ports: - - "8091:8080" + - "8191:8080" volumes: - device1-storage:/var/data/qiot/ environment: @@ -29,7 +29,7 @@ services: - edge - qiotconnect ports: - - "8092:8080" + - "8192:8080" volumes: - device2-storage:/var/data/qiot/ environment: @@ -50,7 +50,7 @@ services: - edge - qiotconnect ports: - - "8093:8080" + - "8193:8080" volumes: - device3-storage:/var/data/qiot/ environment: From 4b18c0f4a9be7abd6c5e862eaafe18c5b7d96b4b Mon Sep 17 00:00:00 2001 From: Andrea Battaglia Date: Tue, 9 Mar 2021 11:59:48 +0100 Subject: [PATCH 08/15] prepare for beta version --- datahub/docker-compose.yaml | 2 +- edge/prod/docker-compose.yaml | 19 ++++++++++ openshift.txt | 69 +++++++++++++++++++---------------- 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index 36c4f0a..24e1ce9 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -56,7 +56,7 @@ services: environment: QUARKUS_PROFILE: "staging" STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" - CERTLOCAL: "true" + CERT_MANAGER_ENABLED: "false" LOGLEVEL: "FINE" qiot-datahub-gas-collector-service: diff --git a/edge/prod/docker-compose.yaml b/edge/prod/docker-compose.yaml index 32b697f..f0097d3 100644 --- a/edge/prod/docker-compose.yaml +++ b/edge/prod/docker-compose.yaml @@ -54,10 +54,29 @@ services: REGISTRATION-SERVICE-URL: "http://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" MQTTPORT: "443" + qiot-edge-service-P-04: + image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 + container_name: "qiot-edge-service-P-04" + restart: "no" + networks: + - edge + ports: + - "8094:8080" + volumes: + - qiot-edge-service-P-04-storage:/var/data/qiot/ + environment: + STATIONSERIAL: "test-station-serialP04" + STATIONNAME: "test-station-P04" + STATIONADDRESS: "New York" + SENSORSURL: "http://qiot-edge-sensors:5000/" + REGISTRATION-SERVICE-URL: "http://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" + MQTTPORT: "443" volumes: qiot-edge-service-P-01-storage: qiot-edge-service-P-02-storage: qiot-edge-service-P-03-storage: + qiot-edge-service-P-04-storage: networks: edge: external: true diff --git a/openshift.txt b/openshift.txt index 0bfe1a2..e89b1e6 100644 --- a/openshift.txt +++ b/openshift.txt @@ -1,6 +1,6 @@ /*** LOCALIZATION ***/ # OK -oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-localization.git --name=localization-service +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0.0.2-java11~https://github.com/qiot-project/qiot-datahub-localization.git --name=localization-service # OK oc patch bc/localization-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' # OK @@ -8,28 +8,34 @@ oc start-build localization-service # OK oc new-app localization-service + + /*** STATION ***/ # OK -oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-station.git --name=station-service +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0.0.2-java11~https://github.com/qiot-project/qiot-datahub-station.git --name=station-service # OK -oc patch bc/station-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' +oc patch bc/station-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"8Gi"}}}}' # OK oc start-build station-service # OK -oc new-app station-service -e DBHOST=mysql -e LOCALIZATION-SERVICE-URL=http://localization-service:8080 +oc new-app station-service -e DBHOST=postgresql -e LOCALIZATION-SERVICE-URL=http://localization-service:8080 + + /*** REGISTRATION ***/ # OK -oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-registration.git#v2 --name=registration-service +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0.0.2-java11~https://github.com/qiot-project/qiot-datahub-registration.git#v2 --name=registration-service # OK -oc patch bc/registration-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' +oc patch bc/registration-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"8Gi"}}}}' # OK oc start-build registration-service # OK -oc new-app registration-service -e STATION-SERVICE-URL=http://station-service:8080 -e CERTLOCAL=true +oc new-app registration-service -e STATION-SERVICE-URL=http://station-service:8080 -e CERT_MANAGER_ENABLED=false -e LOGLEVEL=FINE # OK oc expose svc/registration-service + + /*** GAS COLLECTOR ***/ # OK oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-collector.git \ @@ -45,6 +51,8 @@ oc new-app gas-collector-service \ -e STATION-SERVICE-URL=http://station-service:8080 \ -e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 + + /*** POLLUTION COLLECTOR ***/ # OK oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-collector.git \ @@ -60,13 +68,15 @@ oc new-app pollution-collector-service \ -e STATION-SERVICE-URL=http://station-service:8080 \ -e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 + + /*** GAS STORER ***/ # OK oc new-build registry.access.redhat.com/openjdk/openjdk-11-rhel7~https://github.com/qiot-project/qiot-datahub-storer.git#v2 \ --context-dir=. --name=gas-storer-service \ --build-env='MAVEN_ARGS= -pl qiot-datahub-storer-gas -am' \ - --build-env='MAVEN_S2I_ARTIFACT_DIRS=qiot-datahub-storer-gas/target' \ - --build-env='S2I_SOURCE_DEPLOYMENTS_FILTER=*-runner.jar lib' + --build-env='MAVEN_S2I_ARTIFACT_DIRS=qiot-datahub-storer-gas/target/quarkus-app' \ + --build-env='S2I_SOURCE_DEPLOYMENTS_FILTER=*.jar lib app quarkus' # OK oc new-app gas-storer-service \ -e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 \ @@ -81,8 +91,8 @@ oc new-app gas-storer-service \ oc new-build registry.access.redhat.com/openjdk/openjdk-11-rhel7~https://github.com/qiot-project/qiot-datahub-storer.git#v2 \ --context-dir=. --name=pollution-storer-service \ --build-env='MAVEN_ARGS= -pl qiot-datahub-storer-pollution -am' \ - --build-env='MAVEN_S2I_ARTIFACT_DIRS=qiot-datahub-storer-pollution/target' \ - --build-env='S2I_SOURCE_DEPLOYMENTS_FILTER=*-runner.jar lib' + --build-env='MAVEN_S2I_ARTIFACT_DIRS=qiot-datahub-storer-pollution/target/quarkus-app' \ + --build-env='S2I_SOURCE_DEPLOYMENTS_FILTER=*.jar lib app quarkus' # OK oc new-app pollution-storer-service \ -e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 \ @@ -92,25 +102,22 @@ oc new-app pollution-storer-service \ -e INFLUXDB_DATA_BUCKET_NAME=covid19 \ -e INFLUXDB_TOKEN=JCNTRpJj6AsvqiD7MnRWPZhIrXYpHwusT_Mwvc0I26E6lFJsXNzkaD2SbkIxeeH3TbiakNbbZGm_EKk8dlCCbQ== -/*** IMPORTER ***/ -# OK -oc new-build registry.access.redhat.com/ubi8/openjdk-11~https://github.com/qiot-project/qiot-datahub-importer.git#v2 --name=importer-service -# TODO -oc new-app importer-service -e MONGODBURL=mongodb:27017 -# OK -oc expose svc/importer-service - - - - - - - - - - - - - +/*** HISTORICAL_DATA ***/ +# TODO +oc new-build registry.access.redhat.com/openjdk/openjdk-11-rhel7~https://github.com/qiot-project/qiot-datahub-importer.git#v2 \ + --context-dir=. --name=historical-data-service \ + --build-env='MAVEN_ARGS= -Dquarkus.package.type=fast-jar' \ + --build-env='MAVEN_S2I_ARTIFACT_DIRS=target/quarkus-app' \ + --build-env='S2I_SOURCE_DEPLOYMENTS_FILTER=*.jar lib app quarkus' +# TODO +oc new-app historical-data-service \ + -e DBHOST=postgresql \ + -e INFLUXDB_CONNECTION_STRING=http://timeseries-service-influxdb2 \ + -e INFLUXDB_ORG_ID=qiot \ + -e INFLUXDB_DATA_BUCKET_ID=covid19 \ + -e INFLUXDB_DATA_BUCKET_NAME=covid19 \ + -e INFLUXDB_TOKEN=JCNTRpJj6AsvqiD7MnRWPZhIrXYpHwusT_Mwvc0I26E6lFJsXNzkaD2SbkIxeeH3TbiakNbbZGm_EKk8dlCCbQ== +# OK +oc expose svc/historical-data-service From 9192bd26c400615bda50bb450c0e055a5a04c011 Mon Sep 17 00:00:00 2001 From: Andrea Battaglia Date: Fri, 12 Mar 2021 18:41:56 +0100 Subject: [PATCH 09/15] activemq replaced with eclipse mosquitto --- comp/docker-compose.yaml | 38 ++++++++++++++++++++-------- datahub/docker-compose.yaml | 8 +++--- edge/application/docker-compose.yaml | 18 ++++++++----- openshift.txt | 2 +- 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/comp/docker-compose.yaml b/comp/docker-compose.yaml index da40698..f099028 100644 --- a/comp/docker-compose.yaml +++ b/comp/docker-compose.yaml @@ -1,20 +1,35 @@ version: '3' services: - activemq: - image: vromero/activemq-artemis:2-alpine-latest - container_name: "activemq" +# activemq: +# image: vromero/activemq-artemis:2-alpine-latest +# container_name: "activemq" +# restart: "always" +# networks: +# - datahub +# - qiotconnect +# ports: +# - "8161:8161" +## - "61616:61616" +## - "5672:5672" +# - "1883:1883" +# environment: +# ARTEMIS_USERNAME: "quarkus" +# ARTEMIS_PASSWORD: "quarkus" + + mosquitto: + image: eclipse-mosquitto:1.6.2 + container_name: "mosquitto" restart: "always" networks: - datahub - qiotconnect ports: - - "8161:8161" -# - "61616:61616" -# - "5672:5672" - - "1883:1883" - environment: - ARTEMIS_USERNAME: "quarkus" - ARTEMIS_PASSWORD: "quarkus" + - "1883:1883" + - "9001:9001" +# volumes: +# - mosquitto-storage-config:/mosquitto/config +# - mosquitto-storage-data:/mosquitto/data +# - mosquitto-storage-logs:/mosquitto/log kafka: image: obsidiandynamics/kafka @@ -107,6 +122,9 @@ volumes: mysql-storage: # mongodb-storage: postgres-storage: + mosquitto-storage-data: + mosquitto-storage-logs: + mosquitto-storage-config: networks: datahub: external: true diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index 24e1ce9..85219f9 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -52,7 +52,7 @@ services: expose: - "8080" ports: - - "10001:8080" + - "8443:8443" environment: QUARKUS_PROFILE: "staging" STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" @@ -68,8 +68,7 @@ services: environment: QUARKUS_PROFILE: "staging" STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" - AMQPHOST: "activemq" - MQTTHOST: "activemq" + MQTTHOST: "mosquitto" qiot-datahub-pollution-collector-service: image: quay.io/qiotcovid19/qiot-datahub-collector-pollution:1.0.0-alpha container_name: "qiot-datahub-pollution-collector-service" @@ -79,8 +78,7 @@ services: environment: QUARKUS_PROFILE: "staging" STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" - AMQPHOST: "activemq" - MQTTHOST: "activemq" + MQTTHOST: "mosquitto" qiot-datahub-gas-storer-service: image: quay.io/qiotcovid19/qiot-datahub-storer-gas:2.0.0-alpha diff --git a/edge/application/docker-compose.yaml b/edge/application/docker-compose.yaml index 311fba6..2c364fb 100644 --- a/edge/application/docker-compose.yaml +++ b/edge/application/docker-compose.yaml @@ -17,10 +17,12 @@ services: STATIONNAME: "teststationS01" STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" SENSORSURL: "http://qiot-edge-sensors:5000/" - REGISTRATION-SERVICE-URL: "http://qiot-datahub-registration-service:8080/" - MQTTURL: "activemq" + REGISTRATION-SERVICE-URL: "https://qiot-datahub-registration-service:8443/" + MQTTURL: "mosquitto" ENABLESSL: "false" MQTTPORT: "1883" + LOGLEVEL: "INFO" + qiot-edge-service-02: image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 container_name: "qiot-edge-service-02" @@ -38,10 +40,12 @@ services: STATIONNAME: "teststationS02" STATIONADDRESS: "soriano calabro" SENSORSURL: "http://qiot-edge-sensors:5000/" - REGISTRATION-SERVICE-URL: "http://qiot-datahub-registration-service:8080/" - MQTTURL: "activemq" + REGISTRATION-SERVICE-URL: "https://qiot-datahub-registration-service:8443/" + MQTTURL: "mosquitto" ENABLESSL: "false" MQTTPORT: "1883" + LOGLEVEL: "INFO" + qiot-edge-service-03: image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 container_name: "qiot-edge-service-03" @@ -59,10 +63,12 @@ services: STATIONNAME: "teststationS03" STATIONADDRESS: "minsk" SENSORSURL: "http://qiot-edge-sensors:5000/" - REGISTRATION-SERVICE-URL: "http://qiot-datahub-registration-service:8080/" - MQTTURL: "activemq" + REGISTRATION-SERVICE-URL: "https://qiot-datahub-registration-service:8443/" + MQTTURL: "mosquitto" ENABLESSL: "false" MQTTPORT: "1883" + LOGLEVEL: "INFO" + volumes: device1-storage: device2-storage: diff --git a/openshift.txt b/openshift.txt index e89b1e6..0038261 100644 --- a/openshift.txt +++ b/openshift.txt @@ -32,7 +32,7 @@ oc start-build registration-service # OK oc new-app registration-service -e STATION-SERVICE-URL=http://station-service:8080 -e CERT_MANAGER_ENABLED=false -e LOGLEVEL=FINE # OK -oc expose svc/registration-service +oc create route passthrough registration-service --service=registration-service From fd6603550e00e9159706694fa85088b24b7ad56c Mon Sep 17 00:00:00 2001 From: Andrea Battaglia Date: Tue, 16 Mar 2021 19:45:05 +0100 Subject: [PATCH 10/15] fixed input to adhere to registration-service validation pattern --- datahub/docker-compose.yaml | 1 + edge/application/docker-compose.yaml | 14 +++++------ edge/prod/docker-compose.yaml | 35 ++++++++++++++++++---------- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index 85219f9..20a12e1 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -41,6 +41,7 @@ services: QUARKUS_PROFILE: "staging" DBHOST: "postgresql" LOCALIZATION-SERVICE-URL: "http://qiot-datahub-localization-service:8080" + LOGLEVEL: "FINE" qiot-datahub-registration-service: image: quay.io/qiotcovid19/qiot-datahub-registration:2.0.0-alpha diff --git a/edge/application/docker-compose.yaml b/edge/application/docker-compose.yaml index 2c364fb..03069ed 100644 --- a/edge/application/docker-compose.yaml +++ b/edge/application/docker-compose.yaml @@ -13,9 +13,9 @@ services: - device1-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" - STATIONSERIAL: "teststationserialS01" - STATIONNAME: "teststationS01" - STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" + STATIONSERIAL: "teststationserials01" + STATIONNAME: "teststationserials01" + STATIONADDRESS: "viale john fitzgerald kennedy 2D vibo valentia" SENSORSURL: "http://qiot-edge-sensors:5000/" REGISTRATION-SERVICE-URL: "https://qiot-datahub-registration-service:8443/" MQTTURL: "mosquitto" @@ -36,8 +36,8 @@ services: - device2-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" - STATIONSERIAL: "teststationserialS02" - STATIONNAME: "teststationS02" + STATIONSERIAL: "teststationserials02" + STATIONNAME: "teststations02" STATIONADDRESS: "soriano calabro" SENSORSURL: "http://qiot-edge-sensors:5000/" REGISTRATION-SERVICE-URL: "https://qiot-datahub-registration-service:8443/" @@ -59,8 +59,8 @@ services: - device3-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" - STATIONSERIAL: "teststationserialS03" - STATIONNAME: "teststationS03" + STATIONSERIAL: "teststationserials03" + STATIONNAME: "teststations03" STATIONADDRESS: "minsk" SENSORSURL: "http://qiot-edge-sensors:5000/" REGISTRATION-SERVICE-URL: "https://qiot-datahub-registration-service:8443/" diff --git a/edge/prod/docker-compose.yaml b/edge/prod/docker-compose.yaml index f0097d3..7dbb9bf 100644 --- a/edge/prod/docker-compose.yaml +++ b/edge/prod/docker-compose.yaml @@ -11,13 +11,16 @@ services: volumes: - qiot-edge-service-P-01-storage:/var/data/qiot/ environment: - STATIONSERIAL: "test-station-serialP01" - STATIONNAME: "test-station-P01" + STATIONSERIAL: "test-station-serialp01" + STATIONNAME: "test-station-p01" STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" SENSORSURL: "http://qiot-edge-sensors:5000/" - REGISTRATION-SERVICE-URL: "http://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" MQTTPORT: "443" + ENABLESSL: "true" + LOGLEVEL: "INFO" + qiot-edge-service-P-02: image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 container_name: "qiot-edge-service-P-02" @@ -29,13 +32,16 @@ services: volumes: - qiot-edge-service-P-02-storage:/var/data/qiot/ environment: - STATIONSERIAL: "test-station-serialP02" - STATIONNAME: "test-station-P02" + STATIONSERIAL: "test-station-serialp02" + STATIONNAME: "test-station-p02" STATIONADDRESS: "soriano calabro" SENSORSURL: "http://qiot-edge-sensors:5000/" - REGISTRATION-SERVICE-URL: "http://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" MQTTPORT: "443" + ENABLESSL: "true" + LOGLEVEL: "INFO" + qiot-edge-service-P-03: image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 container_name: "qiot-edge-service-P-03" @@ -47,13 +53,16 @@ services: volumes: - qiot-edge-service-P-03-storage:/var/data/qiot/ environment: - STATIONSERIAL: "test-station-serialP03" - STATIONNAME: "test-station-P03" + STATIONSERIAL: "test-station-serialp03" + STATIONNAME: "test-station-p03" STATIONADDRESS: "minsk" SENSORSURL: "http://qiot-edge-sensors:5000/" - REGISTRATION-SERVICE-URL: "http://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" MQTTPORT: "443" + ENABLESSL: "true" + LOGLEVEL: "INFO" + qiot-edge-service-P-04: image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 container_name: "qiot-edge-service-P-04" @@ -65,13 +74,15 @@ services: volumes: - qiot-edge-service-P-04-storage:/var/data/qiot/ environment: - STATIONSERIAL: "test-station-serialP04" - STATIONNAME: "test-station-P04" + STATIONSERIAL: "test-station-serialp04" + STATIONNAME: "test-station-p04" STATIONADDRESS: "New York" SENSORSURL: "http://qiot-edge-sensors:5000/" - REGISTRATION-SERVICE-URL: "http://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" MQTTPORT: "443" + ENABLESSL: "true" + LOGLEVEL: "INFO" volumes: qiot-edge-service-P-01-storage: qiot-edge-service-P-02-storage: From a42bd7152ec6a45df2104e28341af59aefc7fd57 Mon Sep 17 00:00:00 2001 From: Andrea Battaglia Date: Tue, 23 Mar 2021 14:41:02 +0100 Subject: [PATCH 11/15] minor improvements --- datahub/docker-compose.yaml | 6 +++--- edge/prod/docker-compose.yaml | 2 +- openshift.txt | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index 20a12e1..0a0be24 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -27,7 +27,7 @@ services: - "8080" environment: QUARKUS_PROFILE: "staging" - LOGLEVEL: "FINE" + LOGLEVEL: "INFO" qiot-datahub-station-service: image: quay.io/qiotcovid19/qiot-datahub-station:1.0.0-alpha @@ -41,7 +41,7 @@ services: QUARKUS_PROFILE: "staging" DBHOST: "postgresql" LOCALIZATION-SERVICE-URL: "http://qiot-datahub-localization-service:8080" - LOGLEVEL: "FINE" + LOGLEVEL: "INFO" qiot-datahub-registration-service: image: quay.io/qiotcovid19/qiot-datahub-registration:2.0.0-alpha @@ -58,7 +58,7 @@ services: QUARKUS_PROFILE: "staging" STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" CERT_MANAGER_ENABLED: "false" - LOGLEVEL: "FINE" + LOGLEVEL: "INFO" qiot-datahub-gas-collector-service: image: quay.io/qiotcovid19/qiot-datahub-collector-gas:1.0.0-alpha diff --git a/edge/prod/docker-compose.yaml b/edge/prod/docker-compose.yaml index 7dbb9bf..8fbbf2e 100644 --- a/edge/prod/docker-compose.yaml +++ b/edge/prod/docker-compose.yaml @@ -16,7 +16,7 @@ services: STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" SENSORSURL: "http://qiot-edge-sensors:5000/" REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" - MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" + MQTTURL: "ssl://endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" MQTTPORT: "443" ENABLESSL: "true" LOGLEVEL: "INFO" diff --git a/openshift.txt b/openshift.txt index 0038261..79c30c3 100644 --- a/openshift.txt +++ b/openshift.txt @@ -38,7 +38,7 @@ oc create route passthrough registration-service --service=registration-service /*** GAS COLLECTOR ***/ # OK -oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-collector.git \ +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-collector.git \ --context-dir=. --name=gas-collector-service --build-env='ARTIFACT_DIR=qiot-datahub-collector-gas/target' \ --build-env='MVN_ADDITIONAL_ARGS= -pl qiot-datahub-collector-gas -am' # OK @@ -55,7 +55,7 @@ oc new-app gas-collector-service \ /*** POLLUTION COLLECTOR ***/ # OK -oc new-build quay.io/qiot/ubi-quarkus-native-s2i:20.3.1-java11~https://github.com/qiot-project/qiot-datahub-collector.git \ +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-collector.git \ --context-dir=. --name=pollution-collector-service --build-env='ARTIFACT_DIR=qiot-datahub-collector-pollution/target' \ --build-env='MVN_ADDITIONAL_ARGS= -pl qiot-datahub-collector-pollution -am ' # OK From 25aa99711749b66c7879b0eed0b81e8c6473c4d0 Mon Sep 17 00:00:00 2001 From: Andrea Battaglia Date: Wed, 7 Apr 2021 12:09:27 +0200 Subject: [PATCH 12/15] influxdb2 update and cleanup --- comp/docker-compose.yaml | 69 +++++-------------- .../provisioning/datasources/datasource.yml | 4 +- 2 files changed, 19 insertions(+), 54 deletions(-) diff --git a/comp/docker-compose.yaml b/comp/docker-compose.yaml index f099028..c8fc2e2 100644 --- a/comp/docker-compose.yaml +++ b/comp/docker-compose.yaml @@ -1,20 +1,5 @@ version: '3' services: -# activemq: -# image: vromero/activemq-artemis:2-alpine-latest -# container_name: "activemq" -# restart: "always" -# networks: -# - datahub -# - qiotconnect -# ports: -# - "8161:8161" -## - "61616:61616" -## - "5672:5672" -# - "1883:1883" -# environment: -# ARTEMIS_USERNAME: "quarkus" -# ARTEMIS_PASSWORD: "quarkus" mosquitto: image: eclipse-mosquitto:1.6.2 @@ -26,10 +11,6 @@ services: ports: - "1883:1883" - "9001:9001" -# volumes: -# - mosquitto-storage-config:/mosquitto/config -# - mosquitto-storage-data:/mosquitto/data -# - mosquitto-storage-logs:/mosquitto/log kafka: image: obsidiandynamics/kafka @@ -48,28 +29,6 @@ services: KAFKA_RESTART_DELAY: "5" ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: "0" -# mysql: -# image: "mysql" -# container_name: "mysql" -# command: --default-authentication-plugin=mysql_native_password -# restart: always -# environment: -# MYSQL_ROOT_PASSWORD: "qiot" -# MYSQL_DATABASE: "qiot" -# MYSQL_USER: "qiot" -# MYSQL_PASSWORD: "qiot" -# ports: -# # : < MySQL Port running inside container> -# - '3306:3306' -# expose: -# # Opens port 3306 on the container -# - '3306' -# # Where our data will be persisted -# volumes: -# - mysql-storage:/var/lib/mysql -# security_opt: -# - seccomp:unconfined - postgresql: image: postgres container_name: "postgresql" @@ -91,20 +50,29 @@ services: - "18080:8080" influxdb: - image: "quay.io/influxdb/influxdb:v2.0.3" + image: "influxdb:latest" container_name: "influxdb" ports: - "8086:8086" - "9999:9999" volumes: - - influxdb-storage:/root/.influxdbv2 + - influxdb-user-storage:/root/.influxdbv2 + - influxdb-data-storage:/var/lib/influxdb2 + - influxdb-config-storage:/etc/influxdb2 environment: - INFLUXDB_DB: "qiot" - INFLUXDB_ADMIN_USER: "qiot" - INFLUXDB_ADMIN_PASSWORD: "qiot" + DOCKER_INFLUXDB_INIT_MODE: "setup" + DOCKER_INFLUXDB_INIT_USERNAME: "qiot" + DOCKER_INFLUXDB_INIT_PASSWORD: "qiotcovid19password" + DOCKER_INFLUXDB_INIT_ORG: "qiot" + DOCKER_INFLUXDB_INIT_BUCKET: "covid19" +# DOCKER_INFLUXDB_INIT_RETENTION: The duration the system's initial bucket should retain data. If not set, the initial bucket will retain data forever + DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: "qiotcovid19influxdbtoken" + grafana: image: "grafana/grafana" container_name: "grafana" + links: + - influxdb ports: - "3000:3000" volumes: @@ -117,14 +85,11 @@ services: GF_SECURITY_ADMIN_PASSWORD: "qiot" volumes: - influxdb-storage: + influxdb-user-storage: + influxdb-data-storage: + influxdb-config-storage: grafana-storage: - mysql-storage: -# mongodb-storage: postgres-storage: - mosquitto-storage-data: - mosquitto-storage-logs: - mosquitto-storage-config: networks: datahub: external: true diff --git a/comp/volumes/grafana/provisioning/datasources/datasource.yml b/comp/volumes/grafana/provisioning/datasources/datasource.yml index 6fa4bd6..14ddf50 100644 --- a/comp/volumes/grafana/provisioning/datasources/datasource.yml +++ b/comp/volumes/grafana/provisioning/datasources/datasource.yml @@ -3,9 +3,9 @@ datasources: - name: InfluxDB type: influxdb access: proxy - database: qiotcovid19 + database: qiot user: qiot - password: qiotcovid19 + password: qiot url: http://influxdb:8086 isDefault: true editable: true From 527da3b414a040728afa48500fd17d823d3d410c Mon Sep 17 00:00:00 2001 From: Andrea Battaglia Date: Thu, 8 Apr 2021 11:23:07 +0200 Subject: [PATCH 13/15] datahub compose updated --- buildQiot.sh | 4 ++-- datahub/docker-compose.yaml | 48 ++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/buildQiot.sh b/buildQiot.sh index 37fa83a..881c922 100755 --- a/buildQiot.sh +++ b/buildQiot.sh @@ -16,8 +16,8 @@ chmod +x build*.sh ./buildgas.jvm.sh ./buildpollution.jvm.sh cd ~/git/qiot/datahub/qiot-datahub-importer/ -chmod +x build.jvm.sh -./build.jvm.sh +chmod +x build.sh +./build.sh #cd ~/git/qiot/edge/qiot-sensors-emulator/ #chmod +x build.sh #./build.sh diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index 0a0be24..3c59639 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -1,8 +1,8 @@ version: '3' services: - qiot-datahub-import-service: - image: quay.io/qiotcovid19/qiot-datahub-importer:2.0.0-beta - container_name: "qiot-datahub-import-service" + qiot-datahub-historicaldata-service: + image: quay.io/qiotcovid19/qiot-datahub-historicaldata:2.0.0 + container_name: "qiot-datahub-historicaldata-service" restart: "always" networks: - datahub @@ -11,14 +11,14 @@ services: environment: QUARKUS_PROFILE: "staging" DBHOST: "postgresql" - INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" - INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" - INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" - INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" - INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" +# INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" +# INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" +# INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" +# INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" +# INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" qiot-datahub-localization-service: - image: quay.io/qiotcovid19/qiot-datahub-localization:1.0.0-alpha + image: quay.io/qiotcovid19/qiot-datahub-localization:1.0.0 container_name: "qiot-datahub-localization-service" restart: "always" networks: @@ -30,7 +30,7 @@ services: LOGLEVEL: "INFO" qiot-datahub-station-service: - image: quay.io/qiotcovid19/qiot-datahub-station:1.0.0-alpha + image: quay.io/qiotcovid19/qiot-datahub-station:1.0.0 container_name: "qiot-datahub-station-service" restart: "always" networks: @@ -44,7 +44,7 @@ services: LOGLEVEL: "INFO" qiot-datahub-registration-service: - image: quay.io/qiotcovid19/qiot-datahub-registration:2.0.0-alpha + image: quay.io/qiotcovid19/qiot-datahub-registration:2.0.0 container_name: "qiot-datahub-registration-service" restart: "always" networks: @@ -58,10 +58,10 @@ services: QUARKUS_PROFILE: "staging" STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" CERT_MANAGER_ENABLED: "false" - LOGLEVEL: "INFO" + LOGLEVEL: "FINE" qiot-datahub-gas-collector-service: - image: quay.io/qiotcovid19/qiot-datahub-collector-gas:1.0.0-alpha + image: quay.io/qiotcovid19/qiot-datahub-collector-gas:1.0.0 container_name: "qiot-datahub-gas-collector-service" restart: "always" networks: @@ -71,7 +71,7 @@ services: STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" MQTTHOST: "mosquitto" qiot-datahub-pollution-collector-service: - image: quay.io/qiotcovid19/qiot-datahub-collector-pollution:1.0.0-alpha + image: quay.io/qiotcovid19/qiot-datahub-collector-pollution:1.0.0 container_name: "qiot-datahub-pollution-collector-service" restart: "always" networks: @@ -82,29 +82,29 @@ services: MQTTHOST: "mosquitto" qiot-datahub-gas-storer-service: - image: quay.io/qiotcovid19/qiot-datahub-storer-gas:2.0.0-alpha + image: quay.io/qiotcovid19/qiot-datahub-storer-gas:2.0.0 container_name: "qiot-datahub-gas-storer-service" restart: "always" environment: QUARKUS_PROFILE: "staging" KAFKABOOTSTRAPURL: "kafka:9092" INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" - INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" - INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" - INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" - INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" + INFLUXDB_TOKEN: "qiotcovid19influxdbtoken" + INFLUXDB_ORG_ID: "qiot" + INFLUXDB_DATA_BUCKET_ID: "covid19" + INFLUXDB_DATA_BUCKET_NAME: "covid19" qiot-datahub-pollution-storer-service: - image: quay.io/qiotcovid19/qiot-datahub-storer-pollution:2.0.0-alpha + image: quay.io/qiotcovid19/qiot-datahub-storer-pollution:2.0.0 container_name: "qiot-datahub-pollution-storer-service" restart: "always" environment: QUARKUS_PROFILE: "staging" KAFKABOOTSTRAPURL: "kafka:9092" INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" - INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" - INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" - INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" - INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" + INFLUXDB_TOKEN: "qiotcovid19influxdbtoken" + INFLUXDB_ORG_ID: "qiot" + INFLUXDB_DATA_BUCKET_ID: "covid19" + INFLUXDB_DATA_BUCKET_NAME: "covid19" networks: datahub: external: true From a13adfeab48735e2d074829092ea8781475da6de Mon Sep 17 00:00:00 2001 From: Andrea Battaglia Date: Thu, 8 Apr 2021 17:29:01 +0200 Subject: [PATCH 14/15] updates --- openshift.txt | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/openshift.txt b/openshift.txt index 79c30c3..44fa27d 100644 --- a/openshift.txt +++ b/openshift.txt @@ -1,6 +1,6 @@ /*** LOCALIZATION ***/ # OK -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0.0.2-java11~https://github.com/qiot-project/qiot-datahub-localization.git --name=localization-service +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-localization.git --name=localization-service # OK oc patch bc/localization-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' # OK @@ -12,7 +12,7 @@ oc new-app localization-service /*** STATION ***/ # OK -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0.0.2-java11~https://github.com/qiot-project/qiot-datahub-station.git --name=station-service +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-station.git --name=station-service # OK oc patch bc/station-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"8Gi"}}}}' # OK @@ -24,7 +24,7 @@ oc new-app station-service -e DBHOST=postgresql -e LOCALIZATION-SERVICE-URL=http /*** REGISTRATION ***/ # OK -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0.0.2-java11~https://github.com/qiot-project/qiot-datahub-registration.git#v2 --name=registration-service +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-registration.git#v2 --name=registration-service # OK oc patch bc/registration-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"8Gi"}}}}' # OK @@ -33,7 +33,9 @@ oc start-build registration-service oc new-app registration-service -e STATION-SERVICE-URL=http://station-service:8080 -e CERT_MANAGER_ENABLED=false -e LOGLEVEL=FINE # OK oc create route passthrough registration-service --service=registration-service - +# TEST +oc create configmap override --from-file=registration-ConfigMap.yaml +oc set volumes deploy/registration-service --add --name=config -m /home/quarkus/config --configmap-name=override --overwrite /*** GAS COLLECTOR ***/ @@ -106,18 +108,11 @@ oc new-app pollution-storer-service \ /*** HISTORICAL_DATA ***/ # TODO -oc new-build registry.access.redhat.com/openjdk/openjdk-11-rhel7~https://github.com/qiot-project/qiot-datahub-importer.git#v2 \ - --context-dir=. --name=historical-data-service \ - --build-env='MAVEN_ARGS= -Dquarkus.package.type=fast-jar' \ - --build-env='MAVEN_S2I_ARTIFACT_DIRS=target/quarkus-app' \ - --build-env='S2I_SOURCE_DEPLOYMENTS_FILTER=*.jar lib app quarkus' -# TODO -oc new-app historical-data-service \ - -e DBHOST=postgresql \ - -e INFLUXDB_CONNECTION_STRING=http://timeseries-service-influxdb2 \ - -e INFLUXDB_ORG_ID=qiot \ - -e INFLUXDB_DATA_BUCKET_ID=covid19 \ - -e INFLUXDB_DATA_BUCKET_NAME=covid19 \ - -e INFLUXDB_TOKEN=JCNTRpJj6AsvqiD7MnRWPZhIrXYpHwusT_Mwvc0I26E6lFJsXNzkaD2SbkIxeeH3TbiakNbbZGm_EKk8dlCCbQ== +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-importer.git#v2 \ + --name=historical-data-service +# OK +oc patch bc/historical-data-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"8Gi"}}}}' +# OK +oc start-build historical-data-service # OK -oc expose svc/historical-data-service +oc new-app historical-data-service -e DBHOST=postgresql From e749548d0067b35bcaddff7fbd4f674fc405b06d Mon Sep 17 00:00:00 2001 From: Andrea Battaglia Date: Sat, 17 Apr 2021 17:37:11 +0200 Subject: [PATCH 15/15] container images upgraded to RC --- edge/application/docker-compose.yaml | 6 +++--- edge/prod/docker-compose.yaml | 28 ++++++++++++++-------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/edge/application/docker-compose.yaml b/edge/application/docker-compose.yaml index 03069ed..9977f9b 100644 --- a/edge/application/docker-compose.yaml +++ b/edge/application/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3' services: qiot-edge-service-01: - image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 container_name: "qiot-edge-service-01" restart: "no" networks: @@ -24,7 +24,7 @@ services: LOGLEVEL: "INFO" qiot-edge-service-02: - image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 container_name: "qiot-edge-service-02" restart: "no" networks: @@ -47,7 +47,7 @@ services: LOGLEVEL: "INFO" qiot-edge-service-03: - image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 container_name: "qiot-edge-service-03" restart: "no" networks: diff --git a/edge/prod/docker-compose.yaml b/edge/prod/docker-compose.yaml index 8fbbf2e..bcd8e5e 100644 --- a/edge/prod/docker-compose.yaml +++ b/edge/prod/docker-compose.yaml @@ -1,7 +1,7 @@ version: '3' services: qiot-edge-service-P-01: - image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 container_name: "qiot-edge-service-P-01" restart: "no" networks: @@ -11,18 +11,18 @@ services: volumes: - qiot-edge-service-P-01-storage:/var/data/qiot/ environment: - STATIONSERIAL: "test-station-serialp01" - STATIONNAME: "test-station-p01" + STATIONSERIAL: "test-station-serialp10" + STATIONNAME: "test-station-p10" STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" SENSORSURL: "http://qiot-edge-sensors:5000/" REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" - MQTTURL: "ssl://endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" + MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" MQTTPORT: "443" ENABLESSL: "true" - LOGLEVEL: "INFO" + LOGLEVEL: "FINE" qiot-edge-service-P-02: - image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 container_name: "qiot-edge-service-P-02" restart: "no" networks: @@ -32,8 +32,8 @@ services: volumes: - qiot-edge-service-P-02-storage:/var/data/qiot/ environment: - STATIONSERIAL: "test-station-serialp02" - STATIONNAME: "test-station-p02" + STATIONSERIAL: "test-station-serialp11" + STATIONNAME: "test-station-p11" STATIONADDRESS: "soriano calabro" SENSORSURL: "http://qiot-edge-sensors:5000/" REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" @@ -43,7 +43,7 @@ services: LOGLEVEL: "INFO" qiot-edge-service-P-03: - image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 container_name: "qiot-edge-service-P-03" restart: "no" networks: @@ -53,8 +53,8 @@ services: volumes: - qiot-edge-service-P-03-storage:/var/data/qiot/ environment: - STATIONSERIAL: "test-station-serialp03" - STATIONNAME: "test-station-p03" + STATIONSERIAL: "test-station-serialp12" + STATIONNAME: "test-station-p12" STATIONADDRESS: "minsk" SENSORSURL: "http://qiot-edge-sensors:5000/" REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" @@ -64,7 +64,7 @@ services: LOGLEVEL: "INFO" qiot-edge-service-P-04: - image: quay.io/qiotcovid19/edge-service:2.0.0-alpha-x86_64 + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 container_name: "qiot-edge-service-P-04" restart: "no" networks: @@ -74,8 +74,8 @@ services: volumes: - qiot-edge-service-P-04-storage:/var/data/qiot/ environment: - STATIONSERIAL: "test-station-serialp04" - STATIONNAME: "test-station-p04" + STATIONSERIAL: "test-station-serialp13" + STATIONNAME: "test-station-p13" STATIONADDRESS: "New York" SENSORSURL: "http://qiot-edge-sensors:5000/" REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/"