From 6783305f70057e14869647b095302973472af6e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Sat, 2 Dec 2023 07:38:34 +0100 Subject: [PATCH 1/4] Simplify the development setup --- application/.gitignore | 1 + application/Dockerfile.application-dev | 4 +- application/Makefile | 76 +++++++++++--------------- application/README.md | 66 ++++++++-------------- 4 files changed, 60 insertions(+), 87 deletions(-) diff --git a/application/.gitignore b/application/.gitignore index ef5a5d0ce..71f8f6c1f 100644 --- a/application/.gitignore +++ b/application/.gitignore @@ -14,3 +14,4 @@ test/tira-root/ **/*.received.txt .data-dumps/ +src/tira/static/tira/frontend-vuetify/ diff --git a/application/Dockerfile.application-dev b/application/Dockerfile.application-dev index f7850169b..5b9da8451 100644 --- a/application/Dockerfile.application-dev +++ b/application/Dockerfile.application-dev @@ -72,5 +72,7 @@ RUN cd /tmp-del \ && mv node_modules/.bin /usr/local/lib/node_modules/.bin \ && mv node_modules/.yarn-integrity /usr/local/lib/node_modules/.yarn-integrity \ && cd / \ - && rm -R /tmp-del/ + && rm -R /tmp-del/ \ + && apt-get install -y vim htop bash-completion \ + && echo "\n. /usr/share/bash-completion/bash_completion" >> ~/.bashrc diff --git a/application/Makefile b/application/Makefile index 13972707f..9d11014bf 100644 --- a/application/Makefile +++ b/application/Makefile @@ -1,34 +1,49 @@ .PHONY: help setup setup-webpack webpack run-develop build-docker clean -VENV_NAME?=venv -PYTHON=${PWD}/${VENV_NAME}/bin/python3 VERSION_APPLICATION=0.0.96 VERSION_GRPC=0.0.96 .DEFAULT: help help: @echo "make setup" - @echo " setup venv, compile protobuf, give permissions to tira, and make migrations" + @echo " setup your environment" @echo "make run-develop" - @echo " run server as tira user" + @echo " run the tira server" + @echo "make tests" + @echo " run all tests (automatically done in Github Actions on each commit)" + @echo "make vite-build" + @echo " build and test the frontnend client code" @echo "make build-docker" - @echo " build the docker containers " + @echo " build the docker containers for deployment" @echo "make clean" - @echo " remove setup files" - -setup: + @echo " clean the environment" +clean: rm -f ../model/src/state/tira.sqlite3 rm -f ../model/src/model/virtual-machines/admin.prototext - chmod +x setup.sh - bash setup.sh - chmod o+x src - test -d $(VENV_NAME) || mkdir -p src/config/ - cp config/tira-application-config.dev.yml src/config/settings.yml - make venv - sh -c ". $(VENV_NAME)/bin/activate && \ - ${PYTHON} src/manage.py index_model && \ - ${PYTHON} src/manage.py createcachetable" +setup: + python3 src/manage.py migrate + python3 src/manage.py createcachetable + python3 src/manage.py makemigrations tira + python3 src/manage.py migrate tira + python3 src/manage.py index_model + make vite-build + +tests: + ./test/run_all_tests.sh + +run-develop: + python src/manage.py makemigrations \ + && python src/manage.py migrate --fake \ + && python src/manage.py run_develop + +vite-build: + @cd src/tira/frontend-vuetify \ + && yarn build + +vite-build-light: + @cd src/tira/frontend-vuetify \ + && yarn build-light .data-dumps/django-db-dump.zip: @mkdir -p .data-dumps && \ @@ -54,30 +69,6 @@ import-data-from-dump: .data-dumps/django-db-dump.zip ${PYTHON} src/manage.py loaddata .data-dumps/django-db-dump.json -tests: - ./test/run_all_tests.sh - - -# Requirements are in setup.py, so whenever setup.py is changed, re-run installation of dependencies. -venv: - test -d $(VENV_NAME) || python3 -m venv $(VENV_NAME) - sh -c ". $(VENV_NAME)/bin/activate && \ - python3 -m pip install wheel && \ - python3 -m pip install -r requirements.txt" - -run-develop: - python src/manage.py makemigrations \ - && python src/manage.py migrate --fake \ - && python src/manage.py run_develop - -vite-build: - @cd src/tira/frontend-vuetify \ - && yarn build - -vite-build-light: - @cd src/tira/frontend-vuetify \ - && yarn build-light - vite-build-docker: docker run -v ${PWD}:/app --platform linux/amd64 --rm -ti -w /app/src/tira/frontend-vuetify --entrypoint yarn webis/tira-application:basis-0.0.95 build @@ -119,6 +110,3 @@ docker-publish: build-docker-basis: docker build --tag webis/tira-application:basis-${VERSION_APPLICATION} -f Dockerfile.basis .. -clean: - rm -r $(VENV_NAME) - diff --git a/application/README.md b/application/README.md index aa3293d91..69dc3dcb7 100644 --- a/application/README.md +++ b/application/README.md @@ -1,31 +1,33 @@ +# The TIRA Application + +The backend server and frontend of the TIRA application. + ## Development Setup -The following steps will setup a self-contained, local tira application and a mockup tira host. See [Development](#development) for more detailed options. +We use devcontainers for development. To start your environment, either use Github Codespaces (click on "Code" -> "Codespaces" in Github to open one) as easiest way to get started, or [devpod](https://github.com/loft-sh/devpod) as open source alternative (directly pointing to our Kubernetes or your local docker installation). + +Run `make` to get an overview of all commands that will setup a self-contained tira application in your dev environment. -1. Install Python3, pip, virtualenv, yarn and the mysql tools. For Ubuntu: +1. Setup the database and compile the vuetify frontend ```bash - ~$ sudo apt-get update && sudo apt-get install python3 python3-pip python3-venv libmysqlclient-dev yarn + ~$ make setup ``` -2. Setup the local environment +2. Start the local environment, point your browser to the specified URL ```bash - ~$ make setup # This creates the virtual environment and prepares Django's database + ~$ make run-develop ``` -3. Initialize your development database from a database dump +3. Optionally: To work on real data, initialize your development database from a database dump ```bash ~$ make import-data-from-dump ``` -4. Setup the local environment - ```bash - ~$ make run-develop # This updates the config and runs the server within the venv. - ``` +4. Optionally: Change the configuration (the settings used for the development setup are: `tira/application/config/settings-dev.yml`) -5. Run all unit tests - ```bash - ~$ make tests - ``` +## Frontend Development + +Build the frontend code via `make vite-build` ## Docker @@ -47,45 +49,25 @@ You need to run two docker containers for a tira-application: `registry.webis.de ## Build and Deploy -### Run the tests +### Step 1: Run the tests ```bash - application/src~$ python3 manage.py test test tira/tests/ # run all tests in application/src/tira/tests - application/src~$ python3 manage.py test test tira/tests/tests.py # run an individual test module + make tests # run all tests (automatically done in Github Actions on each commit) ``` -### Deploy on Kubernetes - -- Add the discourse secret in the namespace via: `tira-host/src/tira_scripts/k8s-deploy-discourse-api-key.sh` - -### Re-build the docker images: +### Step 2: Re-build the docker images: ```bash - protocol~$ make build # Build the protobuf libraries from source. - ~$ make setup # This creates the virtual environment and prepares Django's database - ~$ make docker-build-tira-application # Build the docker image (deploy mode with nginx) - ~$ make docker-run-tira-application # Run the docker container with the make command (deploy mode) - ~$ make docker-publish-tira-application # (optional) Publish a new version + ~$ make build-docker + ~$ make build-docker-all ``` These make targets from the deployment configuration: `tira/application/config/settings-deploy.yml` -### Development - -The settings used for the development setup are: `tira/application/config/settings-dev.yml` +### Step 3: Deploy on Kubernetes -Frequently used development commands are: - - ```bash - application/src~$ python3 manage.py runserver 8080 # Start the application without any grpc server - application/src~$ python3 manage.py grpc_server # Start only the application's grpc server - application/src~$ python3 manage.py run_develop # Start the application and the application's grpc server. This is used in make run-develop and the container - application/src~$ python3 manage.py run_mockup # Start the application, the application's grpc server, and a mock host grpc server that will reply to the application with fake commands. This is the simplest way to develop the application. - ``` - -### Frontend Development - -Build the frontend code via `make vite` +- `code-admin-knowledge-base/services/tira/` contains all the deployment yamls. +- Add the discourse secret in the namespace via: `tira-host/src/tira_scripts/k8s-deploy-discourse-api-key.sh` ## Troubleshooting From 7afa158d59121d64e5e359774d6e99ebfa24858f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Sat, 2 Dec 2023 07:50:15 +0100 Subject: [PATCH 2/4] Simplify dev environment --- .github/workflows/run-all-tests.yml | 3 --- application/Makefile | 7 +++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/run-all-tests.yml b/.github/workflows/run-all-tests.yml index b6eaac19b..33a4af3b6 100644 --- a/.github/workflows/run-all-tests.yml +++ b/.github/workflows/run-all-tests.yml @@ -19,9 +19,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run tests run: | - cd application make setup - make setup - . ./venv/bin/activate make tests diff --git a/application/Makefile b/application/Makefile index 9d11014bf..a900f7eef 100644 --- a/application/Makefile +++ b/application/Makefile @@ -1,4 +1,4 @@ -.PHONY: help setup setup-webpack webpack run-develop build-docker clean +.PHONY: help setup run-develop build-docker clean VERSION_APPLICATION=0.0.96 VERSION_GRPC=0.0.96 @@ -17,11 +17,13 @@ help: @echo " build the docker containers for deployment" @echo "make clean" @echo " clean the environment" + clean: rm -f ../model/src/state/tira.sqlite3 rm -f ../model/src/model/virtual-machines/admin.prototext setup: + pip3 install -r requirements.txt python3 src/manage.py migrate python3 src/manage.py createcachetable python3 src/manage.py makemigrations tira @@ -68,14 +70,12 @@ import-data-from-dump: .data-dumps/django-db-dump.zip ${PYTHON} src/manage.py migrate tira && \ ${PYTHON} src/manage.py loaddata .data-dumps/django-db-dump.json - vite-build-docker: docker run -v ${PWD}:/app --platform linux/amd64 --rm -ti -w /app/src/tira/frontend-vuetify --entrypoint yarn webis/tira-application:basis-0.0.95 build vite-build-light-docker: docker run -v ${PWD}:/app --platform linux/amd64 --rm -ti -w /app/src/tira/frontend-vuetify --entrypoint yarn webis/tira-application:basis-0.0.95 build-light - run-docker: docker run -td --rm \ --name=tira-application \ @@ -88,7 +88,6 @@ run-docker: -v=/mnt/ceph/tira:/mnt/ceph/tira \ webis/tira-application-grpc:${VERSION_GRPC} - build-docker: docker build --tag webis/tira-application:${VERSION_APPLICATION} -f Dockerfile.application .. From f7d5a0f476169cb584eb77e8e28c91a3f3b3334f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Sat, 2 Dec 2023 07:51:07 +0100 Subject: [PATCH 3/4] Simplify dev environment --- Makefile | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..88a6672e3 --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +.PHONY: help setup run-develop build-docker clean + +.DEFAULT: help +help: + @echo "make setup" + @echo " setup your environment" + @echo "make run-develop" + @echo " run the tira server" + @echo "make tests" + @echo " run all tests (automatically done in Github Actions on each commit)" + @echo "make vite-build" + @echo " build and test the frontnend client code" + @echo "make clean" + @echo " clean the environment" + + +setup: + @cd application && make setup + +run-develop: + @cd application && make run-develop + +tests: + @cd application && make tests + +vite-build: + @cd application && make vite-build + +clean: + @cd application && make clean + From 7fce22b5bd3ee0c63cc4b8f62599ba4f40b36c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Sat, 2 Dec 2023 08:04:57 +0100 Subject: [PATCH 4/4] improve dev setup --- .github/workflows/run-all-tests.yml | 1 + application/Makefile | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-all-tests.yml b/.github/workflows/run-all-tests.yml index 33a4af3b6..449d19f02 100644 --- a/.github/workflows/run-all-tests.yml +++ b/.github/workflows/run-all-tests.yml @@ -19,6 +19,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run tests run: | + mkdir -p application/src/tira/static/tira/frontend-vuetify/ make setup make tests diff --git a/application/Makefile b/application/Makefile index a900f7eef..51c8e413c 100644 --- a/application/Makefile +++ b/application/Makefile @@ -29,7 +29,7 @@ setup: python3 src/manage.py makemigrations tira python3 src/manage.py migrate tira python3 src/manage.py index_model - make vite-build + make src/tira/static/tira/frontend-vuetify/ tests: ./test/run_all_tests.sh @@ -39,6 +39,9 @@ run-develop: && python src/manage.py migrate --fake \ && python src/manage.py run_develop +src/tira/static/tira/frontend-vuetify/: + @make vite-vuild + vite-build: @cd src/tira/frontend-vuetify \ && yarn build