Merge pull request #229 from introlab/dev

Merge for version 1.2.4

.github/workflows/gen-doc-and-deploy-to-github-pages.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.10.7]
+        python-version: [3.11.4]
 
     steps:
     - uses: actions/checkout@v3
@@ -56,7 +56,7 @@ jobs:
       run: |
         cmake .
         make python-all
-        echo "OPENTERA_PYTHON=$(echo $PWD/python/env/python-3.10/bin/python)" >> $GITHUB_ENV
+        echo "OPENTERA_PYTHON=$(echo $PWD/python/env/python-3.11/bin/python)" >> $GITHUB_ENV
         echo "PYTHONPATH=$(echo $PWD/python)" >> $GITHUB_ENV
 
     - name: Generate Self Signed TLS Certificates

.github/workflows/python-package-pypi.yml

@@ -13,14 +13,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.10.7]
+        python-version: [3.11.4]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         submodules: recursive
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -42,8 +42,8 @@ jobs:
     - name: Build dist package with setuptools
       working-directory: teraserver/python
       run: | 
-        ./env/python-3.10/bin/python3 -m pip install --upgrade setuptools wheel twine
-        ./env/python-3.10/bin/python3 setup.py sdist bdist_wheel
+        ./env/python-3.11/bin/python3 -m pip install --upgrade setuptools wheel twine
+        ./env/python-3.11/bin/python3 setup.py sdist bdist_wheel
         du -h dist
         
     - name: Run Tests
@@ -56,7 +56,7 @@ jobs:
     - name: Upload package to PyPi
       working-directory: teraserver/python
       run: | 
-        ./env/python-3.10/bin/python3 -m twine upload dist/*
+        ./env/python-3.11/bin/python3 -m twine upload dist/*
       env:
         TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/run_tests_on_pull_request.yml

@@ -13,14 +13,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.10.7]
+        python-version: [3.11.4]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         submodules: recursive
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -56,7 +56,7 @@ jobs:
       run: |
         cmake .
         make python-all
-        echo "OPENTERA_PYTHON=$(echo $PWD/python/env/python-3.10/bin/python)" >> $GITHUB_ENV
+        echo "OPENTERA_PYTHON=$(echo $PWD/python/env/python-3.11/bin/python)" >> $GITHUB_ENV
         echo "PYTHONPATH=$(echo $PWD/python)" >> $GITHUB_ENV
     
     - name: Generate Self Signed TLS Certificates
@@ -100,7 +100,12 @@ jobs:
       working-directory: teraserver/python/tests/opentera/db/models
       run: |
         $OPENTERA_PYTHON -m unittest discover . "test_*.py"
-        
+
+    - name: Run Service Tests
+      working-directory: teraserver/python/tests/opentera/services
+      run: |
+        $OPENTERA_PYTHON -m unittest discover . "test_*.py"
+
     - name: Run FileTransferService Tests
       working-directory: teraserver/python/tests/services/FileTransferService
       run: |

.gitignore

@@ -48,13 +48,17 @@ coverage.xml
 
 # Sphinx documentation
 docs/_build/
+docs/--no-check-certificate
+docs/swagger.json
 
 build-teraplus-Desktop_Qt_5_12_0_MSVC2017_64bit-Debug
 *.user
 build-teraserver-Desktop_Qt_5_12_0_MSVC2017_64bit-Debug
 .idea
 python-3.6
 
+# Docker
+docker/prod/certificates/*.pem
 
 # Node modules
 node_modules/
@@ -92,3 +96,4 @@ joss-paper/paper.jats
 joss-paper/paper.pdf
 /.vscode
 _python-3.10
+python-3.11

docker/dev/README.md

@@ -1,52 +1,102 @@
-# Getting Started for Developers - Docker
+# Setup a dev Environment with Docker-Compose on Mac/Linux/Windows
 
-## Pre-requisites
+[Docker Compose](https://docs.docker.com/compose/) is a tool that simplifies the management of multi-container [Docker](https://docs.docker.com/) applications. It allows you to define and run complex applications using a simple YAML file, specifying the services, networks, and volumes needed, making it easier to orchestrate and manage multiple Docker containers as a single application. An example Docker Compose configuration is provided with OpenTeraServer on GitHub in the [docker/dev](https://github.com/introlab/opentera/tree/main/docker/dev) directory.
+
+## Warnings
+
+The following example is provided for testing on a local machine and is not suited for production.
+
+## Requirements
 
 * [Install Docker Desktop for Windows/Linux/Mac](https://www.docker.com/products/docker-desktop/)
 * [Install Visual Studio Code](https://code.visualstudio.com/download)
 * Enough free space on your disk (10Gb+)
 
 ### Install Docker Desktop
 
-* Follow installation procedures and use default settings.
+* Follow installation procedures for your operating system and use default settings. You might need to have Administration rights to proceed with installation.
 
 ### Install Visual Studio Code
 
-* Follow installation procedures and use default settings.
+* Follow installation procedures for your operating system and use default settings. You might need to have Administration rights to proceed with installation.
 
 #### Visual Studio Code Extensions
 
-* Install the following extensions:
-  * [Python](https://marketplace.visualstudio.com/items?itemName=ms-python.python)
-  * [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)
-  * [Docker](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-docker)
-  * [CMake](https://marketplace.visualstudio.com/items?itemName=twxs.cmake)
-  * [Markdown](https://marketplace.visualstudio.com/items?itemName=yzhang.markdown-all-in-one)
+Launch Visual Studio Code and install the following [extensions](https://code.visualstudio.com/docs/editor/extension-marketplace):
+
+* [Python](https://marketplace.visualstudio.com/items?itemName=ms-python.python)
+* [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)
+* [Docker](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-docker)
+* [CMake](https://marketplace.visualstudio.com/items?itemName=twxs.cmake)
+* [Markdown](https://marketplace.visualstudio.com/items?itemName=yzhang.markdown-all-in-one)
 
 ## Open OpenTera Base Folder in Visual Studio Code
 
+We assume here that you have cloned the [code from GitHub](https://github.com/introlab/opentera.git).
 You must select the root directory of the opentera git project.
 
-## Build the Development Environment with docker-compose
+### Build the Development Environment with docker-compose
+
+A sample [docker-compose.yml](https://github.com/introlab/opentera/blob/main/docker/dev/docker-compose.yml) file is provided that generates a full environment with the following components running in individual containers :
+
+* Redis server
+* PostgresSQL server
+* NGINX reverse proxy server
+* A certificate generator for https self-signed certificates
+* TeraServer and base services
+
+Procedure:
+
+1. Right-click on the [docker-compose.yml](https://github.com/introlab/opentera/blob/main/docker/dev/docker-compose.yml) and select "Compose Up". This can take several minutes the first time to download all the images and start the containers.
+2. Docker compose will create **volumes** (shared disks) to store your databases, configurations, files as specified in the "volume:" section. Those directories are automatically mounted when the containers are started.
+3. When running for the first time, databases with default values are generated.
+4. **The default user is "admin" with password "admin".**
+5. The opentera source code is copied (not mounted) at the moment. If you make changes in the code, you need to do "Compose Up" again to generate the new container.
+
+**At this stage you have a running OpenTeraServer on your local machine on port 40075. Congratulations!**
+
+API Documentation can be viewed by browsing to [https://localhost:40075/doc](https://localhost:40075/doc). Accept the security risks if your browser is giving you a warning, this is caused because by your local server using a self generated certificate.
+
+### Connecting to the Local Server with OpenTeraPlus
+
+You can use the GUI application OpenTeraPlus to connect and configure the server. [The application can be downloaded from OpenTeraPlus files release](https://github.com/introlab/openteraplus/releases). Alternatively, you can compile the application. More information are available on [OpenTeraPlus GitHub project](https://github.com/introlab/openteraplus).
+
+#### Configuring OpenTeraPlus with your Local Server
+
+The default OpenTeraPlus configuration should allow you to connect to the server named: "Local - Port 40075". You might want to add "Local-Docker" server in the Documents/OpenTeraPlus/config/OpenTeraPlus.json for OpenTeraPlus if you decide to change your docker-compose configuration with a new port :
+
+```json
+{
+    "Settings": {
+        "showServers": true,
+        "logToFile": true
+    },
+    "Servers": {
+        "Local - Port 40075": {
+                "url": "localhost",
+                "port": 40075
+        },
+        "Local-Docker": {
+                "url": "localhost",
+                "port": 40075
+        }
+    }
+}
+```
 
-A sample [docker-compose.yml](./docker-compose.yml) file is provided that generates a full environment with the following components running in individual containers :
-- Redis server
-- PostgresSQL server 
-- NGINX reverse proxy server 
-- A certificate generator for https self-signed certificates
-- TeraServer and base services
+## Advanced Options
 
->1. Right-click on the [docker-compose.yml](./docker-compose.yml) and select "Compose Up". This can take several minutes the first time to download all the images and start the containers.
+### Connect to the Local Database using pgAdmin
 
->2. Docker compose will create **volumes** (shared disks) to store your databases, configurations, files as specified in the "volume:" section. Those directories are automatically mounted when the containers are started. 
+[pgAdmin](https://www.pgadmin.org/) is an open-source administration and management tool for the PostgreSQL database. It provides a user-friendly graphical interface for interacting with PostgreSQL databases, allowing users to perform tasks like querying the database, managing tables and data, and configuring server settings without needing to use command-line tools.
 
->3. When running for the first time, databases with default values are generated.
+You can connect to the PostgreSQL database running in your container by using port **5433** and using **user: "postgres"** and **password: "postgres"**. This is useful for debugging databases or to understand its structure.
 
->4. The default user is "admin" with password "admin".
+### Connect to Redis
 
->5. The opentera source code is copied (not mounted) at the moment. If you make changes in the code, you need to do "Compose Up" again to generate the new container.
+Redis connexion is available on port **6380**. You can use your favorite tool to inspect the cache and publish/subscribe to topics for debugging.
 
-## Debugging with VS Code
+### Debugging with VS Code
 
 Using the Visual Studio Code remote debugger, you can connect to running code in the opentera-server container on port 5678. A mapping to the local code corresponding to the container TeraServer code is required. You can create a configuration file named "launch.json" in the .vscode directory that contains :
 

docker/dev/certificates/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-bullseye
+FROM python:3.11.5-bullseye
 
 # Install requirements (not changing often)
 ADD ./teraserver/python/env/requirements.txt /requirements.txt

docker/dev/teraserver/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10.10-bullseye
+FROM python:3.11.5-bullseye
 
 # Change default shell to bash
 SHELL ["/bin/bash", "-c"]
@@ -38,15 +38,15 @@ ADD ./teraserver/python/env/requirements.txt /requirements.txt
 RUN ["/bin/bash", "-c", "python3 -m pip install -r /requirements.txt"]
 
 # Install latest npm / nodejs
-RUN curl -sL https://deb.nodesource.com/setup_16.x -o /nodesource_setup.sh
+RUN curl -sL https://deb.nodesource.com/setup_18.x -o /nodesource_setup.sh
 RUN bash /nodesource_setup.sh
 RUN apt update && DEBIAN_FRONTEND=noninteractive apt install -y nodejs
 
 # Add directory from host (path relative to context)
 ADD ./teraserver /teraserver
 
 # Remove local environment that was copied in the ADD command if it exists
-RUN if [ -d "/teraserver/python/env/python-3.10" ]; then rm -rf /teraserver/python/env/python-3.10; fi
+RUN if [ -d "/teraserver/python/env/python-3.11" ]; then rm -rf /teraserver/python/env/python-3.11; fi
 
 # Cleanup directory of cmake cache (we possibly are using it outside of docker)
 RUN if [ -f "/teraserver/CMakeCache.txt" ]; then rm -f /teraserver/CMakeCache.txt; fi

docker/prod/README.md

@@ -0,0 +1,117 @@
+# Deploying with Docker on Linux Servers
+
+This procedure require Linux server management knowledge and adequate permissions. It is provided as an example and can be changed according to your exact requirements and security constraints. OpenTera's authors cannot take any responsability for the use of the software, as mentioned in the [Apache Licence v2](https://www.apache.org/licenses/LICENSE-2.0.txt).
+
+## Warnings
+
+* If you want a test environment or a dev environment, please have a look at the [Developers Setup for Docker documentation](../dev/README.md).
+
+## Requirements
+
+* A Linux server installation.
+* [Docker server edition](https://docs.docker.com/engine/install/) for your favorite Linux distribution.
+* Make sure you setup the system with a user member of the "docker" group.
+* Internet connexion
+
+## Setup with docker-compose
+
+[Docker Compose](https://docs.docker.com/compose/) is a tool that simplifies the management of multi-container [Docker](https://docs.docker.com/) applications. It allows you to define and run complex applications using a simple YAML file, specifying the services, networks, and volumes needed, making it easier to orchestrate and manage multiple Docker containers as a single application.
+
+The next sections will give details of a sample docker-compose setup. Edit and change it to fit your needs.
+
+### Installation Steps
+
+#### Step 1: Create Docker Environment Directory
+
+Create a directory with the name of the environment you want to configure and change directory to the newly created directory. Name the directory as you wish. All the next steps will be done in this directory.
+
+```bash
+mkdir opentera-docker-prod
+cd opentera-docker-prod
+```
+
+#### Step 2: Create `docker-compose.yml` File
+
+Create and edit a file named `docker-compose.yml` with the following [content provided here](docker-compose.yml).
+
+#### Step 3: Create PostgreSQL Configuration file
+
+From your base directory create a directory named `db` and a file named [init.sql](db/init.sql) that will contain all the SQL commands to initialize the databases.
+
+#### Step 4: Create Proxy (NGINX) Configuration Files
+
+From your base directory create a directory named `nginx` and two files named [nginx.conf](nginx/nginx.conf) and [opentera.conf](nginx/opentera.conf). The files are specifically configured to work with Docker.
+
+> NOTE: You might have to copy manually certificates to the `certificates` shared volume.
+
+#### Step 5: Create OpenTera Server Configuration Files
+
+From your base directory create a directory named `teraserver` and five files named [Dockerfile](teraserver/Dockerfile), [FileTransferService.json](teraserver/FileTransferService.json), [LoggingService.json](teraserver/LoggingService.json), [TeraServerConfig.ini](teraserver/TeraServerConfig.ini) and  [VideoRehabService.json](teraserver/VideoRehabService.json).
+
+First create the `teraserver` directory and the `Dockerfile` that will generate the teraserver image from the latest OpenTera source code from GitHub (main branch).
+
+> Please note that this example does not provide a STUN/TURN server. Modify the file [ice_servers.json](teraserver/ice_servers.json) to configure your favorite or custom STUN/TURN servers.
+
+#### Step 6: Copy your SSL Certificates to the Certificates Directory
+
+The shared volume `certificates` is used to read the SSL certificate configuration.
+
+You need the following files to match with [nginx.conf](nginx/nginx.conf) configuration :
+
+1) certificates/site_cert.pem (main certificate)
+2) certificates/site_key.pem (main certificate key)
+3) certificates/ca_cert.pem (CA authority certificate, will be used to sign device certificate)
+
+> Certificate generation / updates could be automated with a certbot or copied / linked directly to your host machine configuration.
+> You can generate a local CA certificate and key with the script [CreateCertificates.py](../../teraserver/python/CreateCertificates.py)
+
+#### Step 7: Launch the Containers with Docker-Compose
+
+Once everything is configured, launch the containers with the following command :
+
+```bash
+# This will create and download all images. This might take few minutes. After, it will launch in daemon mode.
+# Docker will take care of the rest, even after reboot.
+docker compose up -d
+```
+
+Congratulations, you are done!
+
+## Backups
+
+> Make sure you backup your configuration and your volumes regularly.
+
+## Configure server with OpenTeraPlus
+
+### Connecting to the Local Server with OpenTeraPlus
+
+You can use the GUI application OpenTeraPlus to connect and configure the server. [The application can be downloaded from OpenTeraPlus files release](https://github.com/introlab/openteraplus/releases). Alternatively, you can compile the application. More information are available on [OpenTeraPlus GitHub project](https://github.com/introlab/openteraplus).
+
+### Configuring OpenTeraPlus with Your Local Server
+
+You might want to add "opentera-docker-prod" server in the Documents/OpenTeraPlus/config/OpenTeraPlus.json for OpenTeraPlus  :
+
+```json
+{
+    "Settings": {
+        "showServers": true,
+        "logToFile": true
+    },
+    "Servers": {
+        "Local - Port 40075": {
+                "url": "localhost",
+                "port": 40075
+        },
+        "opentera-docker-prod": {
+                "url": "your server name",
+                "port": 40075
+        }
+    }
+}
+```
+
+Use OpenTeraPlus to:
+
+* Change default password for admin
+* Create sites / users / participants / devices
+* Configure services as needed