Merge pull request #566 from tira-io/development

Development

.github/workflows/build-documentation-nightly.yml

@@ -0,0 +1,25 @@
+name: Documentation
+on:
+  push:
+    branches:
+      - development
+
+jobs:
+  build-doc:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Dependencies
+        run: |
+          sudo apt-get install -y python3-pip
+          pip install sphinx furo
+          cd python-client/sphinx
+          sphinx-apidoc -o . ../tira
+          make html
+          touch _build/html/.nojekyll
+      - name: Deploy
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: python-client/sphinx/_build/html
+          target-folder: nightly

.github/workflows/build-documentation-production.yml

@@ -0,0 +1,27 @@
+name: Documentation
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-doc:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Dependencies
+        run: |
+          sudo apt-get install -y python3-pip
+          pip install sphinx furo
+          cd python-client/sphinx
+          sphinx-apidoc -o . ../tira
+          make html
+          touch _build/html/.nojekyll
+      - name: Deploy
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: python-client/sphinx/_build/html
+          target-folder: ./
+          clean-exclude:
+            - nightly

README.md

@@ -12,34 +12,22 @@ Components:
 
 ## Setup Your Local Development Environment
 
+We use [dev containers](https://code.visualstudio.com/docs/devcontainers/containers) to simplify development. Please install Docker and an IDE with support for dev containers on your machine (we usually use VS Code).
+
 First, please clone the repository:
 ```
 git clone [email protected]:tira-io/tira.git
 ```
 
-Please change your directory to `application`:
-```
-cd application
-```
-
-Install your virtual environment via:
-```
-make setup
-```
+Please open the directory `application` in VS Code, and confirm to use the provided dev container.
 
 If you want to work on production data, please ensure that you can login to ssh.webis.de, and then do the following:
 
 ```
 make import-data-from-dump
 ```
 
-Then, to start TIRA locally, please start:
-
-```
-make webpack-watch
-```
-
-and 
+To start TIRA locally, please run:
 
 ```
 make run-develop

application/.devcontainer.json

@@ -0,0 +1,8 @@
+{
+  "image": "webis/tira-application:basis-0.0.93",
+  "customizations": {
+    "vscode": {
+      "extensions": ["ms-python.python", "ms-python.vscode-pylance", "ms-toolsai.jupyter"]
+    }
+  }
+}

application/Dockerfile.application

@@ -9,7 +9,7 @@ RUN cd /src/tira/frontend-vuetify \
 
 
 # Only change in case of new / updated dependencies
-FROM webis/tira-application:basis-0.0.76
+FROM webis/tira-application:basis-0.0.93
 
 # This Dockerfile ensures that all dependencies do rarely change by starting from a basis image
 # that contains already all dependencies (so that the minor versions do rarely change, but we
@@ -19,19 +19,13 @@ COPY application /tira/application
 
 WORKDIR /tira/application/src
 
-RUN cd /tira/application/src/tira/frontend/ && \
-	ln -s /tira-dependencies/node_modules/ node_modules && \ 
-	npm run build && \
-	mkdir -p /tira/model/src && \
-	cd /tira/application/src && \
+RUN cd /tira/application/src && \
 	chown tira:tira -R /tira/application && \
 	python3 manage.py collectstatic && \
 	chmod +x /tira/application/src/tira/endpoints/aha && \
 	cp /tira/application/src/tira/management/commands/irds_cli.sh /irds_cli.sh && \
 	rm -f ./config/settings.yml ./config/config.yml ./config/tira-application-config.dev.yml && \
-	rm -Rf tira/static/tira/frontend-vuetify/ && rm -Rf /tira/application/src/tira/static/tira/frontend-vuetify/ && \
-	apk add sudo && \
-	echo '%podman ALL=(ALL) ALL' > /etc/sudoers.d/podman
+	rm -Rf tira/static/tira/frontend-vuetify/ && rm -Rf /tira/application/src/tira/static/tira/frontend-vuetify/
 
 COPY --from=0 /src/tira/static/tira/frontend-vuetify/ /tira/application/src/tira/static/tira/frontend-vuetify/
 

application/Dockerfile.application-dev

@@ -1,14 +1,53 @@
-# docker build -t webis/tira-application:basis-0.0.76 -f f.Dockerfile .
-FROM webis/tira-application:basis-0.0.75
-
-ENV PATH="${PATH}:/root/.cargo/bin"
-
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > foo.sh && \
-	sh ./foo.sh -y && \
-	rm foo.sh && \
-	pip3 install coverage coverage-badge && \
-	apk add libffi-dev && \
-	pip3 uninstall -y discourse-client-in-disraptor && \
-	pip3 uninstall -y PyGithub && \
-	pip3 install discourse-client-in-disraptor PyGithub==1.59.1
+# docker build -t webis/tira-application:basis-0.0.93 -f Dockerfile.application-dev .
+FROM ubuntu:22.04
+
+RUN apt-get update \
+	&& apt-get upgrade -y \
+	&& apt-get install -y python3 python3-pip python-is-python3 mysql-client nginx sudo podman git \
+	&& apt-get install -y gcc default-libmysqlclient-dev pkg-config \
+	&& rm -Rf /var/cache/apt \
+	&& rm -Rf /root/.cache/pip \
+	&& echo '%podman ALL=(ALL) ALL' > /etc/sudoers.d/podman
+
+ADD requirements.txt /requirements.txt
+
+RUN pip3 install -r /requirements.txt \
+	&& rm -Rf /var/cache/apt \
+	&& rm -Rf /root/.cache/pip
+
+RUN pip3 uninstall -y diffir \
+	&& pip3 install git+https://github.com/mam10eks/diffir && \
+	addgroup --gid 1010 tira && \
+	adduser --disabled-password --uid 1010 --ingroup tira tira && \
+	adduser tira root && \
+	mkdir /tira-tmp-debug && \
+	mkdir -p /tira/application/src/tira && \
+	mkdir -p /etc/nginx/logs && \
+	mkdir -p /mnt/ceph/tira && \
+	mkdir -p /root/.ir_datasets \
+	&& rm -Rf /var/cache/apt \
+	&& rm -Rf /root/.cache/pip \
+	&& apt-get install -y fuse-overlayfs
+
+COPY src/tira_nginx.conf /etc/nginx/nginx.conf
+
+RUN chown -R tira:tira /usr/local && \
+	chmod 777 /var/run/ && \
+	echo "tira ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers &&\
+	touch /var/log/tira_debug.txt &&\
+	touch /tira/application/src/debug.log &&\
+	chown tira:tira /var/log/tira_debug.txt &&\
+	chown tira:tira /tira/application/src/debug.log &&\
+	chown -R tira:tira /root/.ir_datasets &&\
+	chmod 777 /root/.ir_datasets &&\
+	chmod 777 /root &&\
+	mkdir -p /home/tira/.tira &&\
+	mkdir -p /mnt/ceph/tira/log/tira-application &&\
+	mkdir /home/tira/.ssh &&\
+	mkdir /mnt/ceph/tira/state &&\
+	mkdir -p /usr/share/nginx/logs/ && \
+	chmod 777 /usr/share/nginx/logs/ && \
+	touch /mnt/ceph/tira/state/tira_vm_states.sqlite3 &&\
+	chown tira:tira -R /home/tira &&\
+	chown tira:tira -R /tira
 

application/Makefile

@@ -2,8 +2,8 @@
 
 VENV_NAME?=venv
 PYTHON=${PWD}/${VENV_NAME}/bin/python3
-VERSION_APPLICATION=0.0.92
-VERSION_GRPC=0.0.92
+VERSION_APPLICATION=0.0.93
+VERSION_GRPC=0.0.93
 
 .DEFAULT: help
 help:
@@ -70,9 +70,8 @@ src/tira/frontend/webpack-stats.json:
 	npm install; \
 	npm run build
 
-tests: venv src/tira/frontend/webpack-stats.json
-	@. ./venv/bin/activate \
-	&& ./test/run_all_tests.sh
+tests:
+	./test/run_all_tests.sh
 
 
 # Requirements are in setup.py, so whenever setup.py is changed, re-run installation of dependencies.
@@ -82,10 +81,10 @@ venv:
 		python3 -m pip install wheel && \
 		python3 -m pip install -r requirements.txt"
 
-run-develop: venv
-	$(VENV_NAME)/bin/python src/manage.py makemigrations \
-	&& $(VENV_NAME)/bin/python src/manage.py migrate --fake \
-	&& $(VENV_NAME)/bin/python src/manage.py run_develop
+run-develop:
+	python src/manage.py makemigrations \
+	&& python src/manage.py migrate --fake \
+	&& python src/manage.py run_develop
 
 vite-build:
 	docker run -v ${PWD}:/app --platform linux/amd64 --rm -ti -w /app/src/tira/frontend-vuetify --entrypoint yarn webis/tira:vuetify-dev-0.0.1 build

application/requirements.txt

@@ -14,9 +14,10 @@ python-gitlab
 GitPython
 python-slugify
 git+https://github.com/allenai/ir_datasets
+git+https://github.com/mam10eks/diffir
 pandas
 markdown
-PyGithub
+PyGithub==1.59.1
 ghapi
 parameterized
 mockito

application/src/django_admin/settings.py

@@ -246,7 +246,6 @@ def logger_config(log_dir: Path):
     'medium-resources': {'cores': 2, 'ram': 20, 'gpu': 0, 'data': 'no', 'description': 'Medium (2 CPU Cores, 20GB of RAM)', 'key': 'medium-resources'},
     'large-resources': {'cores': 4, 'ram': 40, 'gpu': 0, 'data': 'no', 'description': 'Large (4 CPU Cores, 40GB of RAM)', 'key': 'large-resources'},
     'small-resources-gpu': {'cores': 1, 'ram': 10, 'gpu': '1-nvidia-1080', 'data': 'no', 'description': 'Small w. GPU (1 CPU Cores, 10GB of RAM, 1 Nvidia GTX 1080 with 8GB)', 'key': 'small-resources-gpu'},
-    'irds-data-mounted': {'cores': 1, 'ram': 10, 'gpu': '0', 'data': 'irds', 'description': 'Small (1 CPU Cores, 10GB of RAM, IRDS)', 'key': 'irds-data-mounted'},
 }
 
 DEFAULT_GIT_INTEGRATION_URL = 'https://git.webis.de/code-research/tira'

application/src/tira/data/HybridDatabase.py

@@ -8,6 +8,7 @@
 import randomname
 import os
 import zipfile
+import json
 
 from tira.util import TiraModelWriteError, TiraModelIntegrityError
 from tira.proto import TiraClientWebMessages_pb2 as modelpb
@@ -993,7 +994,8 @@ def get_evaluations_with_keys_by_dataset(self, dataset_id, include_unpublished=F
             tira_evaluation_review.blinded, tira_run_review.published, tira_run_review.blinded,
             tira_evaluation.measure_key, tira_evaluation.measure_value, tira_run_review.reviewer_id, 
             tira_run_review.no_errors, tira_run_review.has_errors, tira_run_review.has_no_errors,
-            tira_evaluation_review.reviewer_id, tira_run_review.reviewer_id
+            tira_evaluation_review.reviewer_id, tira_run_review.reviewer_id, 
+            tira_linktosoftwaresubmissiongitrepository.build_environment
         FROM
             tira_run as evaluation_run
         INNER JOIN 
@@ -1004,6 +1006,8 @@ def get_evaluations_with_keys_by_dataset(self, dataset_id, include_unpublished=F
             tira_software ON input_run.software_id = tira_software.id
         LEFT JOIN
             tira_dockersoftware ON input_run.docker_software_id = tira_dockersoftware.docker_software_id
+        LEFT JOIN
+            tira_linktosoftwaresubmissiongitrepository ON tira_dockersoftware.docker_software_id = tira_linktosoftwaresubmissiongitrepository.docker_software_id
         LEFT JOIN
             tira_review as tira_evaluation_review ON evaluation_run.run_id = tira_evaluation_review.run_id
         LEFT JOIN
@@ -1033,7 +1037,21 @@ def get_evaluations_with_keys_by_dataset(self, dataset_id, include_unpublished=F
         return self.__parse_submissions(rows, include_unpublished, round_floats, show_only_unreviewed, show_only_unreviewed)
 
     @staticmethod
-    def __parse_submissions(rows, include_unpublished, round_floats, include_without_evaluation=False, show_only_unreviewed=False):
+    def __link_to_code(build_environment):
+        if not build_environment:
+            return None
+
+        build_environment = json.loads(build_environment)
+
+        if 'TIRA_JUPYTER_NOTEBOOK' not in build_environment or 'GITHUB_REPOSITORY' not in build_environment or 'GITHUB_WORKFLOW' not in build_environment or 'GITHUB_SHA' not in build_environment:
+            return None
+
+        if build_environment['GITHUB_WORKFLOW'] == ".github/workflows/upload-notebook-submission.yml":
+            return f'https://github.com/{build_environment["GITHUB_REPOSITORY"]}/tree/{build_environment["GITHUB_SHA"]}/jupyter-notebook-submissions/{build_environment["TIRA_JUPYTER_NOTEBOOK"]}'
+
+        return None
+
+    def __parse_submissions(self, rows, include_unpublished, round_floats, include_without_evaluation=False, show_only_unreviewed=False):
         keys = dict()
         input_run_to_evaluation = {}
 
@@ -1047,7 +1065,7 @@ def round_if_float(fl):
 
         for dataset_id, run_id, input_run_id, upload_display_name, upload_vm_id, software_vm_id, docker_display_name, \
                 docker_vm_id, eval_published, eval_blinded, run_published, run_blinded, m_key, m_value, \
-                reviewer_id, no_errors, has_errors, has_no_errors, tira_evaluation_reviewer_id, tira_run_reviewer_id in rows:
+                reviewer_id, no_errors, has_errors, has_no_errors, tira_evaluation_reviewer_id, tira_run_reviewer_id, build_environment in rows:
 
             if (not include_without_evaluation and not m_key) or (not include_unpublished and not eval_published):
                 continue
@@ -1088,6 +1106,7 @@ def round_if_float(fl):
             input_run_to_evaluation[run_id]['is_upload'] = is_upload
             input_run_to_evaluation[run_id]['is_software'] = is_software
             input_run_to_evaluation[run_id]['review_state'] = review_state
+            input_run_to_evaluation[run_id]['link_to_code'] = self.__link_to_code(build_environment)
 
             if m_key:
                 input_run_to_evaluation[run_id]['measures'][m_key] = m_value

application/src/tira/endpoints/diffir_api.py

@@ -14,15 +14,14 @@
 
 
 def doc_file_for_run(vm_id, dataset_id, task_id, run_id):
+    checked_paths = []
     for evaluation in model.get_evaluations_of_run(vm_id, run_id):
-        p = Path(settings.TIRA_ROOT) / "data" / "runs" / dataset_id / vm_id / evaluation / 'output' /\
-                                       ".data-top-10-for-rendering.jsonl"
-        print(p)
-
-        if p.is_file():
-            return p
-
-    raise ValueError('.data-top-10-for-rendering.jsonl')
+        for f in [".data-top-10-for-rendering.jsonl.gz", ".data-top-10-for-rendering.jsonl", ".data-top-10-for-rendering.json.gz", ".data-top-10-for-rendering.json"]:
+            p = Path(settings.TIRA_ROOT) / "data" / "runs" / dataset_id / vm_id / evaluation / 'output' / f
+            checked_paths += [str(p)]
+            if p.is_file():
+                return p
+    raise ValueError(f'Could not find .data-top-10-for-rendering.jsonl. Searched in {checked_paths}.')
 
 
 def load_irds_metadata_of_task(task, dataset):