diff --git a/README.md b/README.md index 8e29d50..bcedf5b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ## qiot-datahub-docker A set od docker-compose files to run the QIoT environment on your host machine using a OCI-compliant container engine and tools. -Podman engine is recommendedthus you'll find below the prerequisited and mandatory actions to take before using these compose files: +Podman engine is recommended, thus you'll find below the prerequisites and mandatory actions to take before using these compose files: # if you want to go with podman: @@ -9,11 +9,10 @@ Podman engine is recommendedthus you'll find below the prerequisited and mandato # Set up a dedicated container network for both your edge pods and datahub pods - `$ sudo docker network create edge` - Network dedicated to the services running on the edge device /etc/cni/net.d/qiot-covid19-edge.conflist -`$ sudo podman network create datahub` - Network dedicated to the services running on the datacenter +`$ sudo docker network create datahub` - Network dedicated to the services running on the datacenter /etc/cni/net.d/qiot-covid19-datahub.conflist `$ sudo docker network create qiotconnect` - Network dedicated to the interaction between the edge device and the datahub @@ -33,7 +32,7 @@ Podman: Podman Cockpit extension We can edit the `/etc/selinux/config` text file with our persistent setting, either enforcing, permissive, or disabled. By default this file appears as shown below. -``` +```properties This file controls the state of SELinux on the system. SELINUX= can take one of these three values: enforcing - SELinux security policy is enforced. @@ -47,4 +46,55 @@ ELINUXTYPE= can take one of three two values: SELINUXTYPE=targeted ``` -We can simply edit the SELINUX variable between enforcing, permissive, or disabled, as outlined in the comments of the file. After editing the file the changes will not be in place immediately and will only change after system reboot. \ No newline at end of file +We can simply edit the SELINUX variable between enforcing, permissive, or disabled, as outlined in the comments of the file. After editing the file the changes will not be in place immediately and will only change after system reboot. + +# The QIoT-covid19 system + +The QIoT covid19 use-case implementation is made up of 3 main layers: + + - **comp**: the software infrastructure, composite of integration products to make micro-services interact with each other and to manage endpoints and data + - **datahub**: micro-services implementing the use-case business logic + - **edge**: software running at the edge device + - **sensors**: a micro-service emulating the behavior of the sensor board connected to the raspberry pi + - **edge**: the micro-service implementing the edge device business logic + +## Spinning up the environment + +Just enter each and every folder and run a docker compose or, alternatively, use Portainer.io to manage your stacks. + +Please follow the order below to make sure all the components are capable to find their dependencies accordingly: + +### software architecture + +```bash + $ cd comp + $ docker-compose up --remove-orphans +``` + +​ access the mysql instance and execute the script contained in the file *mysql-setup.sql*: + + +```bash + $ docker exec -it mysql bash -l +``` + +### business logic + +```bash + $ cd datahub + $ docker-compose up --remove-orphans +``` + +### edge device - sensors + +```bash + $ cd edge/sensors + $ docker-compose up --remove-orphans +``` + +### edge device - business logic + +```bash + $ cd edge/application + $ docker-compose up --remove-orphans +``` \ No newline at end of file diff --git a/buildQiot.sh b/buildQiot.sh index 8c447c8..881c922 100755 --- a/buildQiot.sh +++ b/buildQiot.sh @@ -1,28 +1,27 @@ -cd /home/abattagl/git/qiot/datahub/qiot-datahub-storer/qiot-datahub-storer-gas/ +cd ~/git/qiot/datahub/qiot-datahub-localization/ chmod +x build.sh ./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-storer/qiot-datahub-storer-pollution/ +cd ~/git/qiot/datahub/qiot-datahub-station/ chmod +x build.sh ./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-aggregation/ +cd ~/git/qiot/datahub/qiot-datahub-registration/ chmod +x build.sh ./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-query/ +cd ~/git/qiot/datahub/qiot-datahub-collector/ +chmod +x build*.sh +./buildgas.sh +./buildpollution.sh +cd ~/git/qiot/datahub/qiot-datahub-storer/ +chmod +x build*.sh +./buildgas.jvm.sh +./buildpollution.jvm.sh +cd ~/git/qiot/datahub/qiot-datahub-importer/ chmod +x build.sh ./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-importer/ -chmod +x build.sh -./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-registration/ -chmod +x build.sh -./build.sh -cd /home/abattagl/git/qiot/datahub/qiot-datahub-endpoint/qiot-datahub-endpoint-mqtt/ -chmod +x build.sh -./build.sh -#cd /home/abattagl/git/qiot/edge/qiot-sensors-emulator/ +#cd ~/git/qiot/edge/qiot-sensors-emulator/ #chmod +x build.sh #./build.sh -#cd /home/abattagl/git/qiot/edge/qiot-service/ +#cd ~/git/qiot/edge/qiot-service/ #chmod +x build.sh #./build.sh diff --git a/comp/.gitignore b/comp/.gitignore index eaf90e9..e69de29 100644 --- a/comp/.gitignore +++ b/comp/.gitignore @@ -1 +0,0 @@ -/volumes/ diff --git a/comp/docker-compose.yaml b/comp/docker-compose.yaml index b7ae018..c8fc2e2 100644 --- a/comp/docker-compose.yaml +++ b/comp/docker-compose.yaml @@ -1,40 +1,26 @@ version: '3' services: - activemq: - image: vromero/activemq-artemis:2-alpine-latest - container_name: "activemq" - restart: "no" + + mosquitto: + image: eclipse-mosquitto:1.6.2 + container_name: "mosquitto" + restart: "always" networks: - datahub - qiotconnect ports: - - "8161:8161" -# - "61616:61616" -# - "5672:5672" - - "1883:1883" - environment: - ARTEMIS_USERNAME: "quarkus" - ARTEMIS_PASSWORD: "quarkus" -# kafdrop: -# image: obsidiandynamics/kafdrop -# container_name: "kafkadrop" -# restart: "no" -# ports: -# - "9000:9000" -# environment: -# KAFKA_BROKERCONNECT: "kafka:29092" -# JVM_OPTS: "-Xms16M -Xmx48M -Xss180K -XX:-TieredCompilation -XX:+UseStringDeduplication -noverify" -# depends_on: -# - "kafka" + - "1883:1883" + - "9001:9001" + kafka: image: obsidiandynamics/kafka container_name: "kafka" - restart: "no" + restart: "always" ports: - "2181:2181" - "9092:9092" environment: - KAFKA_LISTENERS: "INTERNAL://:29092,EXTERNAL://kafka:9092" + KAFKA_LISTENERS: "INTERNAL://:29092,EXTERNAL://:9092" KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka:29092,EXTERNAL://kafka:9092" KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT" KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL" @@ -42,38 +28,68 @@ services: KAFKA_RESTART_ATTEMPTS: "10" KAFKA_RESTART_DELAY: "5" ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL: "0" - mongo: - image: "mongo" - container_name: "mongo" + + postgresql: + image: postgres + container_name: "postgresql" + restart: always + ports: + - "5432:5432" + volumes: + - postgres-storage:/var/lib/postgresql/data environment: - MONGO_INITDB_DATABASE: "qiot" - #MONGO_INITDB_ROOT_USERNAME: "root" - #MONGO_INITDB_ROOT_PASSWORD: "root" + POSTGRES_USER: "qiot" + POSTGRES_PASSWORD: "qiot" + POSTGRES_DB: "qiot" + + postgresqladminer: + image: adminer + container_name: "postgresqladminer" + restart: always + ports: + - "18080:8080" + + influxdb: + image: "influxdb:latest" + container_name: "influxdb" + ports: + - "8086:8086" + - "9999:9999" volumes: - #- ./init-mongo.js: /docker-entrypoint-initdb.d/init-mongo.js:ro - - ./volumes/mongo:/data/db + - influxdb-user-storage:/root/.influxdbv2 + - influxdb-data-storage:/var/lib/influxdb2 + - influxdb-config-storage:/etc/influxdb2 + environment: + DOCKER_INFLUXDB_INIT_MODE: "setup" + DOCKER_INFLUXDB_INIT_USERNAME: "qiot" + DOCKER_INFLUXDB_INIT_PASSWORD: "qiotcovid19password" + DOCKER_INFLUXDB_INIT_ORG: "qiot" + DOCKER_INFLUXDB_INIT_BUCKET: "covid19" +# DOCKER_INFLUXDB_INIT_RETENTION: The duration the system's initial bucket should retain data. If not set, the initial bucket will retain data forever + DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: "qiotcovid19influxdbtoken" + + grafana: + image: "grafana/grafana" + container_name: "grafana" + links: + - influxdb ports: - - "27017-27019:27017-27019" -# mongo-express: -# image: mongo-express -# container_name: mongo-express -# environment: -# ME_CONFIG_MONGODB_SERVER: "mongo" -# ME_CONFIG_MONGODB_PORT: "27017" -# #ME_CONFIG_MONGODB_ENABLE_ADMIN: "false" -# #ME_CONFIG_MONGODB_AUTH_DATABASE: "qiot" -# #ME_CONFIG_MONGODB_AUTH_USERNAME: "qiot" -# #ME_CONFIG_MONGODB_AUTH_PASSWORD: "qiot" -# ports: -# - "8081:8081" -# influx: -# image: "quay.io/influxdb/influxdb:v2.0.3" -# container_name: "influx" -# volumes: -# - ./volumes/influxdb2:/root/.influxdbv2 -# ports: -# - "8086:8086" -# - "9999:9999" + - "3000:3000" + volumes: + - grafana-storage:/var/lib/grafana + - ./volumes/grafana/provisioning:/etc/grafana/provisioning + depends_on: + - influxdb + environment: + GF_SECURITY_ADMIN_USER: "qiot" + GF_SECURITY_ADMIN_PASSWORD: "qiot" + +volumes: + influxdb-user-storage: + influxdb-data-storage: + influxdb-config-storage: + grafana-storage: + postgres-storage: networks: datahub: external: true diff --git a/comp/init-mongo.js b/comp/init-mongo.js deleted file mode 100644 index 2ed4c44..0000000 --- a/comp/init-mongo.js +++ /dev/null @@ -1,15 +0,0 @@ -#docker exec -it mongoContainer mongo -use qiot -db.createUser( - { - user: "qiot", - pwd: "qiot", - roles: [ { role: "readWrite", db: "qiot" } ] - } -) -db.counters.insert( - { - _id: "userid", - seq: NumberInt(0) - } -) diff --git a/comp/volumes/grafana/provisioning/dashboards/dashboard.yml b/comp/volumes/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000..d093eea --- /dev/null +++ b/comp/volumes/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +providers: +- name: InfluxDB + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/comp/volumes/grafana/provisioning/datasources/datasource.yml b/comp/volumes/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000..14ddf50 --- /dev/null +++ b/comp/volumes/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,11 @@ +apiVersion: 1 +datasources: + - name: InfluxDB + type: influxdb + access: proxy + database: qiot + user: qiot + password: qiot + url: http://influxdb:8086 + isDefault: true + editable: true diff --git a/datahub/docker-compose.yaml b/datahub/docker-compose.yaml index 67b0b08..3c59639 100644 --- a/datahub/docker-compose.yaml +++ b/datahub/docker-compose.yaml @@ -1,62 +1,110 @@ version: '3' services: - datahub-registration: - image: quay.io/qiotcovid19/qiot-datahub-registration:1.0.1 - container_name: "datahub-registration" - restart: "no" + qiot-datahub-historicaldata-service: + image: quay.io/qiotcovid19/qiot-datahub-historicaldata:2.0.0 + container_name: "qiot-datahub-historicaldata-service" + restart: "always" networks: - datahub - - qiotconnect ports: - - "5016:5016" + - "5100:8080" environment: QUARKUS_PROFILE: "staging" - HTTPPORT: "5016" - MONGODBURL: "mongo:27017" - datahub-aggregation: - image: quay.io/qiotcovid19/qiot-datahub-aggregation:1.0.1 - container_name: "datahub-aggregation" - restart: "no" - ports: - - "5018:5018" + DBHOST: "postgresql" +# INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" +# INFLUXDB_TOKEN: "hBta3AUlJMCb7y4qpG4o1LNGR6CQpbgJi88NYMX2ta8zj5Xi_EBIezhgtPcYNTXhnbKfgcpUbHDgnArIEKxFxA==" +# INFLUXDB_ORG_ID: "08ebf5cf8aef6f0b" +# INFLUXDB_DATA_BUCKET_ID: "f073a47029310613" +# INFLUXDB_DATA_BUCKET_NAME: "qiotcovid19" + + qiot-datahub-localization-service: + image: quay.io/qiotcovid19/qiot-datahub-localization:1.0.0 + container_name: "qiot-datahub-localization-service" + restart: "always" + networks: + - datahub + expose: + - "8080" environment: QUARKUS_PROFILE: "staging" - HTTPPORT: "5018" - MONGODBURL: "mongo:27017" - datahub-query: - image: quay.io/qiotcovid19/qiot-datahub-query:1.0.1 - container_name: "datahub-query" - restart: "no" + LOGLEVEL: "INFO" + + qiot-datahub-station-service: + image: quay.io/qiotcovid19/qiot-datahub-station:1.0.0 + container_name: "qiot-datahub-station-service" + restart: "always" + networks: + - datahub + expose: + - "8080" + environment: + QUARKUS_PROFILE: "staging" + DBHOST: "postgresql" + LOCALIZATION-SERVICE-URL: "http://qiot-datahub-localization-service:8080" + LOGLEVEL: "INFO" + + qiot-datahub-registration-service: + image: quay.io/qiotcovid19/qiot-datahub-registration:2.0.0 + container_name: "qiot-datahub-registration-service" + restart: "always" + networks: + - datahub + - qiotconnect + expose: + - "8080" ports: - - "5200:5200" + - "8443:8443" + environment: + QUARKUS_PROFILE: "staging" + STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" + CERT_MANAGER_ENABLED: "false" + LOGLEVEL: "FINE" + + qiot-datahub-gas-collector-service: + image: quay.io/qiotcovid19/qiot-datahub-collector-gas:1.0.0 + container_name: "qiot-datahub-gas-collector-service" + restart: "always" + networks: + - datahub environment: QUARKUS_PROFILE: "staging" - HTTPPORT: "5200" - MONGODBURL: "mongo:27017" - endpoint-mqtt: - image: quay.io/qiotcovid19/qiot-datahub-endpoint-mqtt:1.0.1 - container_name: "datahub-endpoint-mqtt" - restart: "no" + STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" + MQTTHOST: "mosquitto" + qiot-datahub-pollution-collector-service: + image: quay.io/qiotcovid19/qiot-datahub-collector-pollution:1.0.0 + container_name: "qiot-datahub-pollution-collector-service" + restart: "always" + networks: + - datahub environment: QUARKUS_PROFILE: "staging" - AMQPHOST: "activemq" - MQTTHOST: "activemq" - gasstorer: - image: quay.io/qiotcovid19/qiot-datahub-storer-gas:1.0.1 - container_name: "datahub-storer-gas" - restart: "no" + STATION-SERVICE-URL: "http://qiot-datahub-station-service:8080" + MQTTHOST: "mosquitto" + + qiot-datahub-gas-storer-service: + image: quay.io/qiotcovid19/qiot-datahub-storer-gas:2.0.0 + container_name: "qiot-datahub-gas-storer-service" + restart: "always" environment: QUARKUS_PROFILE: "staging" KAFKABOOTSTRAPURL: "kafka:9092" - MONGODBURL: "mongo:27017" - pollutionstorer: - image: quay.io/qiotcovid19/qiot-datahub-storer-pollution:1.0.1 - container_name: "datahub-storer-pollution" - restart: "no" + INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" + INFLUXDB_TOKEN: "qiotcovid19influxdbtoken" + INFLUXDB_ORG_ID: "qiot" + INFLUXDB_DATA_BUCKET_ID: "covid19" + INFLUXDB_DATA_BUCKET_NAME: "covid19" + qiot-datahub-pollution-storer-service: + image: quay.io/qiotcovid19/qiot-datahub-storer-pollution:2.0.0 + container_name: "qiot-datahub-pollution-storer-service" + restart: "always" environment: QUARKUS_PROFILE: "staging" KAFKABOOTSTRAPURL: "kafka:9092" - MONGODBURL: "mongo:27017" + INFLUXDB_CONNECTION_STRING: "http://influxdb:8086" + INFLUXDB_TOKEN: "qiotcovid19influxdbtoken" + INFLUXDB_ORG_ID: "qiot" + INFLUXDB_DATA_BUCKET_ID: "covid19" + INFLUXDB_DATA_BUCKET_NAME: "covid19" networks: datahub: external: true diff --git a/edge-server/application/docker-compose.yaml b/edge-server/application/docker-compose.yaml deleted file mode 100644 index cd90211..0000000 --- a/edge-server/application/docker-compose.yaml +++ /dev/null @@ -1,35 +0,0 @@ -version: '3' -services: - edge-integrator: - image: quay.io/qiotcovid19/qiot-edge-integrator:1-x86_64 - container_name: "edge-integrator" - restart: "no" - environment: - AMQPHOST: "edge-messaging" - AMQPPORT: 5672 - AMQPUSERNAME: "qiot" - AMQPPASSWORD: "qiot" - SENSORSURL: "http://edge-sensors:5000/" - edge-server-01: - image: quay.io/qiotcovid19/qiot-edge-server:1-x86_64 - container_name: "edge-server-01" - restart: "no" - ports: - - "8091:8080" - environment: - AMQPHOST: "edge-messaging" - AMQPPORT: 5672 - AMQPUSERNAME: "qiot" - AMQPPASSWORD: "qiot" - STATIONSERIAL: "teststationserial01" - STATIONNAME: "teststation01" - STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" - SENSORSURL: "http://edge-sensors:5000/" - DATAHUBURL: "http://datahub-registration:5016/" - MQTTURL: "activemq" - ENABLESSL: "false" - MQTTPORT: "1883" -networks: - default: - external: - name: edge \ No newline at end of file diff --git a/edge-server/sensors/docker-compose.yaml b/edge-server/sensors/docker-compose.yaml deleted file mode 100644 index 6d187dc..0000000 --- a/edge-server/sensors/docker-compose.yaml +++ /dev/null @@ -1,39 +0,0 @@ -version: '3' -services: - edge-messaging: - image: rabbitmq:3 - container_name: edge-messaging - volumes: -# - ./volumes/rabbitmq/etc/:/etc/rabbitmq/ - - ./volumes/rabbitmq/data/:/var/lib/rabbitmq/ - - ./volumes/rabbitmq/logs/:/var/log/rabbitmq/ - - ./volumes/rabbitmq/enable_plugins:/etc/rabbitmq/enabled_plugins - environment: -# RABBITMQ_ERLANG_COOKIE: ${RABBITMQ_ERLANG_COOKIE} - RABBITMQ_DEFAULT_USER: qiot - RABBITMQ_DEFAULT_PASS: qiot - ports: - - 5673:5672 - - 15672:15672 - expose: - - "5672" - edge-sensors: - image: quay.io/qiotcovid19/qiot-sensors-emulator - container_name: "edge-sensors" - restart: "no" - ports: - - "5000:5000" - environment: - QUARKUS_PROFILE: "staging" - edge-sensors: - image: quay.io/qiotcovid19/qiot-sensors-emulator - container_name: "edge-sensors" - restart: "no" - ports: - - "5000:5000" - environment: - QUARKUS_PROFILE: "staging" -networks: - default: - external: - name: edge \ No newline at end of file diff --git a/edge-server/sensors/volumes/rabbitmq/.gitignore b/edge-server/sensors/volumes/rabbitmq/.gitignore deleted file mode 100644 index 82f0c3a..0000000 --- a/edge-server/sensors/volumes/rabbitmq/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/data/ diff --git a/edge-server/sensors/volumes/rabbitmq/enable_plugins b/edge-server/sensors/volumes/rabbitmq/enable_plugins deleted file mode 100644 index c5f2ce4..0000000 --- a/edge-server/sensors/volumes/rabbitmq/enable_plugins +++ /dev/null @@ -1 +0,0 @@ -[rabbitmq_amqp1_0,rabbitmq_management,rabbitmq_web_dispatch,rabbitmq_management_agent]. diff --git a/edge/application/docker-compose.yaml b/edge/application/docker-compose.yaml index 61f5683..9977f9b 100644 --- a/edge/application/docker-compose.yaml +++ b/edge/application/docker-compose.yaml @@ -1,62 +1,78 @@ version: '3' services: - edge-service-01: - image: quay.io/qiotcovid19/edge-service:1.2.0-x86_64 - container_name: "edge-service-01" + qiot-edge-service-01: + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 + container_name: "qiot-edge-service-01" restart: "no" networks: - edge - qiotconnect ports: - - "8091:8080" + - "8191:8080" + volumes: + - device1-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" - STATIONSERIAL: "teststationserial01" - STATIONNAME: "teststation01" - STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" - SENSORSURL: "http://edge-sensors:5000/" - DATAHUBURL: "http://datahub-registration:5016/" - MQTTURL: "activemq" + STATIONSERIAL: "teststationserials01" + STATIONNAME: "teststationserials01" + STATIONADDRESS: "viale john fitzgerald kennedy 2D vibo valentia" + SENSORSURL: "http://qiot-edge-sensors:5000/" + REGISTRATION-SERVICE-URL: "https://qiot-datahub-registration-service:8443/" + MQTTURL: "mosquitto" ENABLESSL: "false" MQTTPORT: "1883" - edge-service-02: - image: quay.io/qiotcovid19/edge-service:1.2.0-x86_64 - container_name: "edge-service-02" + LOGLEVEL: "INFO" + + qiot-edge-service-02: + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 + container_name: "qiot-edge-service-02" restart: "no" networks: - edge - qiotconnect ports: - - "8092:8080" + - "8192:8080" + volumes: + - device2-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" - STATIONSERIAL: "teststationserial02" - STATIONNAME: "teststation02" + STATIONSERIAL: "teststationserials02" + STATIONNAME: "teststations02" STATIONADDRESS: "soriano calabro" - SENSORSURL: "http://edge-sensors:5000/" - DATAHUBURL: "http://datahub-registration:5016/" - MQTTURL: "activemq" + SENSORSURL: "http://qiot-edge-sensors:5000/" + REGISTRATION-SERVICE-URL: "https://qiot-datahub-registration-service:8443/" + MQTTURL: "mosquitto" ENABLESSL: "false" MQTTPORT: "1883" - edge-service-03: - image: quay.io/qiotcovid19/edge-service:1.2.0-x86_64 - container_name: "edge-service-03" + LOGLEVEL: "INFO" + + qiot-edge-service-03: + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 + container_name: "qiot-edge-service-03" restart: "no" networks: - edge - qiotconnect ports: - - "8093:8080" + - "8193:8080" + volumes: + - device3-storage:/var/data/qiot/ environment: QUARKUS_PROFILE: "staging" - STATIONSERIAL: "teststationserial03" - STATIONNAME: "teststation03" + STATIONSERIAL: "teststationserials03" + STATIONNAME: "teststations03" STATIONADDRESS: "minsk" - SENSORSURL: "http://edge-sensors:5000/" - DATAHUBURL: "http://datahub-registration:5016/" - MQTTURL: "activemq" + SENSORSURL: "http://qiot-edge-sensors:5000/" + REGISTRATION-SERVICE-URL: "https://qiot-datahub-registration-service:8443/" + MQTTURL: "mosquitto" ENABLESSL: "false" MQTTPORT: "1883" + LOGLEVEL: "INFO" + +volumes: + device1-storage: + device2-storage: + device3-storage: networks: edge: external: true diff --git a/edge/prod/docker-compose.yaml b/edge/prod/docker-compose.yaml index e04f8df..bcd8e5e 100644 --- a/edge/prod/docker-compose.yaml +++ b/edge/prod/docker-compose.yaml @@ -1,51 +1,93 @@ version: '3' services: - edge-service-prod-01: - image: quay.io/qiotcovid19/edge-service:1.2.0 - container_name: "edge-service-prod-01" + qiot-edge-service-P-01: + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 + container_name: "qiot-edge-service-P-01" restart: "no" + networks: + - edge ports: - "8091:8080" + volumes: + - qiot-edge-service-P-01-storage:/var/data/qiot/ environment: - STATIONSERIAL: "teststationserial01" - STATIONNAME: "teststation01" + STATIONSERIAL: "test-station-serialp10" + STATIONNAME: "test-station-p10" STATIONADDRESS: "viale john fitzgerald kennedy 2/D vibo valentia" - SENSORSURL: "http://edge-sensors:5000/" - #DATAHUBURL: "http://datahub-registration:5016/" - #MQTTHOST: "activemq" - #ENABLESSL: "false" - #MQTTPORT: "1883" - edge-service-prod-02: - image: quay.io/qiotcovid19/edge-service:1.2.0 - container_name: "edge-service-prod-02" + SENSORSURL: "http://qiot-edge-sensors:5000/" + REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" + MQTTPORT: "443" + ENABLESSL: "true" + LOGLEVEL: "FINE" + + qiot-edge-service-P-02: + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 + container_name: "qiot-edge-service-P-02" restart: "no" + networks: + - edge ports: - "8092:8080" + volumes: + - qiot-edge-service-P-02-storage:/var/data/qiot/ environment: - STATIONSERIAL: "teststationserial02" - STATIONNAME: "teststation02" + STATIONSERIAL: "test-station-serialp11" + STATIONNAME: "test-station-p11" STATIONADDRESS: "soriano calabro" - SENSORSURL: "http://edge-sensors:5000/" - #DATAHUBURL: "http://datahub-registration:5016/" - #MQTTHOST: "activemq" - #ENABLESSL: "false" - #MQTTPORT: "1883" - edge-service-prod-03: - image: quay.io/qiotcovid19/edge-service:1.2.0 - container_name: "edge-service-prod-03" + SENSORSURL: "http://qiot-edge-sensors:5000/" + REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" + MQTTPORT: "443" + ENABLESSL: "true" + LOGLEVEL: "INFO" + + qiot-edge-service-P-03: + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 + container_name: "qiot-edge-service-P-03" restart: "no" + networks: + - edge ports: - "8093:8080" + volumes: + - qiot-edge-service-P-03-storage:/var/data/qiot/ environment: - STATIONSERIAL: "teststationserial03" - STATIONNAME: "teststation03" + STATIONSERIAL: "test-station-serialp12" + STATIONNAME: "test-station-p12" STATIONADDRESS: "minsk" - SENSORSURL: "http://edge-sensors:5000/" - #DATAHUBURL: "http://datahub-registration:5016/" - #MQTTHOST: "activemq" - #ENABLESSL: "false" - #MQTTPORT: "1883" + SENSORSURL: "http://qiot-edge-sensors:5000/" + REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" + MQTTPORT: "443" + ENABLESSL: "true" + LOGLEVEL: "INFO" + + qiot-edge-service-P-04: + image: quay.io/qiotcovid19/edge-service:2.0.0-RC-x86_64 + container_name: "qiot-edge-service-P-04" + restart: "no" + networks: + - edge + ports: + - "8094:8080" + volumes: + - qiot-edge-service-P-04-storage:/var/data/qiot/ + environment: + STATIONSERIAL: "test-station-serialp13" + STATIONNAME: "test-station-p13" + STATIONADDRESS: "New York" + SENSORSURL: "http://qiot-edge-sensors:5000/" + REGISTRATION-SERVICE-URL: "https://registration-service-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com/" + MQTTURL: "endpoint-service-mqtts-0-svc-rte-qiotcovid19v2.apps.cluster-fcd8.fcd8.example.opentlc.com" + MQTTPORT: "443" + ENABLESSL: "true" + LOGLEVEL: "INFO" +volumes: + qiot-edge-service-P-01-storage: + qiot-edge-service-P-02-storage: + qiot-edge-service-P-03-storage: + qiot-edge-service-P-04-storage: networks: - default: - external: - name: edge \ No newline at end of file + edge: + external: true diff --git a/edge/sensors/docker-compose.yaml b/edge/sensors/docker-compose.yaml index 56ec6cc..47fa464 100644 --- a/edge/sensors/docker-compose.yaml +++ b/edge/sensors/docker-compose.yaml @@ -1,8 +1,8 @@ version: '3' services: - edge-sensors: + qiot-edge-sensors: image: quay.io/qiotcovid19/edge-sensors-emulator:1.0.1-x86_64 - container_name: "edge-sensors" + container_name: "qiot-edge-sensors" restart: "no" ports: - "5000:5000" diff --git a/influxdbtest/docker-compose.yaml b/influxdbtest/docker-compose.yaml deleted file mode 100644 index 6c6179f..0000000 --- a/influxdbtest/docker-compose.yaml +++ /dev/null @@ -1,36 +0,0 @@ -version: '3' -services: - influxdb: - image: influxdb:1.8.2 - container_name: "influxdb" - ports: - - "9999:9999" - - "8086:8086" - volumes: - - ./volumes/influxdb:/data/db - - chronograph: - image: quay.io/influxdb/chronograf:1.8.6 - container_name: "chronograph" - entrypoint: 'chronograf --kapacitor-url=http://kapacitor:9092 --influxdb-url=http://influxdb:8086' - depends_on: - - "influxdb" - - "kapacitor" - ports: - - "8888:8888" - volumes: - - ./volumes/influxdb:/var/lib/chronograf - - kapacitor: - image: kapacitor - container_name: "kapacitor" - depends_on: - - "influxdb" - environment: - - KAPACITOR_INFLUXDB_0_URLS_0=http://influxdb:8086 - - KAPACITOR_HOSTNAME=kapacitor -networks: - default: - external: - name: influxdb - #9340172063e48e0f5473ff47b5b05403c03d2f43c0b107d710dd3e014b966b06 \ No newline at end of file diff --git a/openshift.txt b/openshift.txt index b7e70cc..44fa27d 100644 --- a/openshift.txt +++ b/openshift.txt @@ -1,45 +1,118 @@ +/*** LOCALIZATION ***/ +# OK +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-localization.git --name=localization-service +# OK +oc patch bc/localization-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' +# OK +oc start-build localization-service +# OK +oc new-app localization-service + + + +/*** STATION ***/ +# OK +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-station.git --name=station-service +# OK +oc patch bc/station-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"8Gi"}}}}' +# OK +oc start-build station-service +# OK +oc new-app station-service -e DBHOST=postgresql -e LOCALIZATION-SERVICE-URL=http://localization-service:8080 + + + /*** REGISTRATION ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-registration.git --name=qiot-registration -oc patch bc/qiot-registration -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-registration -oc new-app qiot-registration -e MONGODBURL=mongodb:27017 -oc expose svc/qiot-registration - -/*** IMPORTER ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-importer.git --name=qiot-importer -oc patch bc/qiot-importer -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-importer -oc new-app qiot-importer -e MONGODBURL=mongodb:27017 -oc expose svc/qiot-importer - -/*** MQTT ENDPOINT ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-endpoint.git --context-dir=qiot-datahub-endpoint-mqtt --name=qiot-endpoint-mqtt -oc patch bc/qiot-endpoint-mqtt -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-endpoint-mqtt -oc new-app qiot-endpoint-mqtt -e MQTTHOST=qiot-broker-hdls-svc -e KAFKABOOTSTRAPURL=kafka-cluster-kafka-bootstrap:9092 +# OK +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-registration.git#v2 --name=registration-service +# OK +oc patch bc/registration-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"8Gi"}}}}' +# OK +oc start-build registration-service +# OK +oc new-app registration-service -e STATION-SERVICE-URL=http://station-service:8080 -e CERT_MANAGER_ENABLED=false -e LOGLEVEL=FINE +# OK +oc create route passthrough registration-service --service=registration-service +# TEST +oc create configmap override --from-file=registration-ConfigMap.yaml +oc set volumes deploy/registration-service --add --name=config -m /home/quarkus/config --configmap-name=override --overwrite + + +/*** GAS COLLECTOR ***/ +# OK +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-collector.git \ + --context-dir=. --name=gas-collector-service --build-env='ARTIFACT_DIR=qiot-datahub-collector-gas/target' \ + --build-env='MVN_ADDITIONAL_ARGS= -pl qiot-datahub-collector-gas -am' +# OK +oc patch bc/gas-collector-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' +# OK +oc start-build gas-collector-service +# TODO +oc new-app gas-collector-service \ +-e MQTTHOST=endpoint-service-hdls-svc \ +-e STATION-SERVICE-URL=http://station-service:8080 \ +-e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 + + + +/*** POLLUTION COLLECTOR ***/ +# OK +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-collector.git \ + --context-dir=. --name=pollution-collector-service --build-env='ARTIFACT_DIR=qiot-datahub-collector-pollution/target' \ + --build-env='MVN_ADDITIONAL_ARGS= -pl qiot-datahub-collector-pollution -am ' +# OK +oc patch bc/pollution-collector-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' +# OK +oc start-build pollution-collector-service +# TODO +oc new-app pollution-collector-service \ +-e MQTTHOST=endpoint-service-hdls-svc \ +-e STATION-SERVICE-URL=http://station-service:8080 \ +-e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 + + /*** GAS STORER ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-storer.git --context-dir=qiot-datahub-storer-gas --name=qiot-storer-gas -oc patch bc/qiot-storer-gas -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-storer-gas -oc new-app qiot-storer-gas -e MONGODBURL=mongodb:27017 -e KAFKABOOTSTRAPURL=kafka-cluster-kafka-bootstrap:9092 +# OK +oc new-build registry.access.redhat.com/openjdk/openjdk-11-rhel7~https://github.com/qiot-project/qiot-datahub-storer.git#v2 \ + --context-dir=. --name=gas-storer-service \ + --build-env='MAVEN_ARGS= -pl qiot-datahub-storer-gas -am' \ + --build-env='MAVEN_S2I_ARTIFACT_DIRS=qiot-datahub-storer-gas/target/quarkus-app' \ + --build-env='S2I_SOURCE_DEPLOYMENTS_FILTER=*.jar lib app quarkus' +# OK +oc new-app gas-storer-service \ + -e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 \ + -e INFLUXDB_CONNECTION_STRING=http://timeseries-service-influxdb2 \ + -e INFLUXDB_ORG_ID=qiot \ # 2d67bc8b65cc1c19 + -e INFLUXDB_DATA_BUCKET_ID=covid19 \ # 507dfe460a44c268 + -e INFLUXDB_DATA_BUCKET_NAME=covid19 \ + -e INFLUXDB_TOKEN=JCNTRpJj6AsvqiD7MnRWPZhIrXYpHwusT_Mwvc0I26E6lFJsXNzkaD2SbkIxeeH3TbiakNbbZGm_EKk8dlCCbQ== /*** POLLUTION STORER ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-storer.git --context-dir=qiot-datahub-storer-pollution --name=qiot-storer-pollution -oc patch bc/qiot-storer-pollution -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-storer-pollution -oc new-app qiot-storer-pollution -e MONGODBURL=mongodb:27017 -e KAFKABOOTSTRAPURL=kafka-cluster-kafka-bootstrap:9092 - -/*** AGGREGATION ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-aggregation.git --name=qiot-aggregation -oc patch bc/qiot-aggregation -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-aggregation -oc new-app qiot-aggregation -e MONGODBURL=mongodb:27017 -oc expose svc/qiot-aggregation - -/*** QUERY ***/ -oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:20.1.0-java11~https://github.com/andreabattaglia/qiot-datahub-query.git --name=qiot-query -oc patch bc/qiot-query -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"6Gi"}}}}' -oc start-build qiot-query -oc new-app qiot-query -e MONGODBURL=mongodb:27017 -oc expose svc/qiot-query \ No newline at end of file +# OK +oc new-build registry.access.redhat.com/openjdk/openjdk-11-rhel7~https://github.com/qiot-project/qiot-datahub-storer.git#v2 \ + --context-dir=. --name=pollution-storer-service \ + --build-env='MAVEN_ARGS= -pl qiot-datahub-storer-pollution -am' \ + --build-env='MAVEN_S2I_ARTIFACT_DIRS=qiot-datahub-storer-pollution/target/quarkus-app' \ + --build-env='S2I_SOURCE_DEPLOYMENTS_FILTER=*.jar lib app quarkus' +# OK +oc new-app pollution-storer-service \ + -e KAFKABOOTSTRAPURL=stream-service-kafka-bootstrap:9092 \ + -e INFLUXDB_CONNECTION_STRING=http://timeseries-service-influxdb2 \ + -e INFLUXDB_ORG_ID=qiot \ + -e INFLUXDB_DATA_BUCKET_ID=covid19 \ + -e INFLUXDB_DATA_BUCKET_NAME=covid19 \ + -e INFLUXDB_TOKEN=JCNTRpJj6AsvqiD7MnRWPZhIrXYpHwusT_Mwvc0I26E6lFJsXNzkaD2SbkIxeeH3TbiakNbbZGm_EKk8dlCCbQ== + + + +/*** HISTORICAL_DATA ***/ +# TODO +oc new-build quay.io/quarkus/ubi-quarkus-native-s2i:21.0-java11~https://github.com/qiot-project/qiot-datahub-importer.git#v2 \ + --name=historical-data-service +# OK +oc patch bc/historical-data-service -p '{"spec":{"resources":{"limits":{"cpu":"4", "memory":"8Gi"}}}}' +# OK +oc start-build historical-data-service +# OK +oc new-app historical-data-service -e DBHOST=postgresql