From cf2c91dfccc32e4a855ea33c1c38672abd55796f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Tue, 21 Nov 2023 11:59:41 +0100 Subject: [PATCH 01/18] Fix unit tests --- python-client/Dockerfile.dev | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python-client/Dockerfile.dev b/python-client/Dockerfile.dev index dd2eb06a0..d45e3ba43 100644 --- a/python-client/Dockerfile.dev +++ b/python-client/Dockerfile.dev @@ -1,4 +1,4 @@ -#docker build -t webis/tira:python-client-dev-0.0.2 -f Dockerfile.dev . +#docker build -t webis/tira:python-client-dev-0.0.3 -f Dockerfile.dev . FROM ubuntu:latest RUN apt-get update \ @@ -25,5 +25,7 @@ RUN echo 'root:1234' | chpasswd # Create new user: "dev" with password "1234" and change to that user RUN useradd -ms /bin/bash dev \ && echo 'dev:1234' | chpasswd \ - && usermod -aG sudo dev + && usermod -aG sudo dev \ + && groupadd -g 973 docker \ + && usermod -aG docker dev USER dev From 5fded39cacfb1b696c7e709e562f070f7191091a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Tue, 21 Nov 2023 12:00:17 +0100 Subject: [PATCH 02/18] Fix unit tests --- python-client/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-client/Makefile b/python-client/Makefile index a8bb8f239..547defc70 100644 --- a/python-client/Makefile +++ b/python-client/Makefile @@ -6,7 +6,7 @@ build-pypi-package: run-tests twine upload dist/tira-0.0.88-py3-none-any.whl dist/tira-0.0.88.tar.gz run-tests: - docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/app -w /app --entrypoint pytest webis/tira:python-client-dev-0.0.2 + docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/app -w /app --entrypoint pytest webis/tira:python-client-dev-0.0.3 run-tests-legacy: ../application/venv/bin/coverage run --data-file=tests/test-coverage/.coverage ../application/venv/bin/pytest \ From d9ac8f8096c45e3f34e518c4e499986d1852e085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Tue, 21 Nov 2023 12:32:38 +0100 Subject: [PATCH 03/18] mf --- application/src/django_admin/settings.py | 1 - application/src/tira/model.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/application/src/django_admin/settings.py b/application/src/django_admin/settings.py index 1542acc3a..4fa6107a6 100644 --- a/application/src/django_admin/settings.py +++ b/application/src/django_admin/settings.py @@ -246,7 +246,6 @@ def logger_config(log_dir: Path): 'medium-resources': {'cores': 2, 'ram': 20, 'gpu': 0, 'data': 'no', 'description': 'Medium (2 CPU Cores, 20GB of RAM)', 'key': 'medium-resources'}, 'large-resources': {'cores': 4, 'ram': 40, 'gpu': 0, 'data': 'no', 'description': 'Large (4 CPU Cores, 40GB of RAM)', 'key': 'large-resources'}, 'small-resources-gpu': {'cores': 1, 'ram': 10, 'gpu': '1-nvidia-1080', 'data': 'no', 'description': 'Small w. GPU (1 CPU Cores, 10GB of RAM, 1 Nvidia GTX 1080 with 8GB)', 'key': 'small-resources-gpu'}, - 'irds-data-mounted': {'cores': 1, 'ram': 10, 'gpu': '0', 'data': 'irds', 'description': 'Small (1 CPU Cores, 10GB of RAM, IRDS)', 'key': 'irds-data-mounted'}, } DEFAULT_GIT_INTEGRATION_URL = 'https://git.webis.de/code-research/tira' diff --git a/application/src/tira/model.py b/application/src/tira/model.py index c8117b718..253703e5e 100644 --- a/application/src/tira/model.py +++ b/application/src/tira/model.py @@ -238,7 +238,7 @@ class SoftwareSubmissionGitRepository(models.Model): class LinkToSoftwareSubmissionGitRepository(models.Model): - docker_software = models.ForeignKey(DockerSoftware, on_delete=models.RESTRICT) + docker_software = models.ForeignKey(DockerSoftware, on_delete=models.CASCADE) software_submission_git_repository = models.ForeignKey(SoftwareSubmissionGitRepository, on_delete=models.RESTRICT) commit_hash = models.CharField(max_length=100) link_to_file = models.CharField(max_length=500) From 19d7a42882702714a2c2a67835eb04d0b0516aba Mon Sep 17 00:00:00 2001 From: Tim Hagen Date: Tue, 21 Nov 2023 11:49:01 +0000 Subject: [PATCH 04/18] install python dependencys without root and converted ir_dataset_test to unittest.TestCase --- python-client/Dockerfile.dev | 7 +- python-client/tests/ir_datasets_test.py | 741 ++++++++++++------------ 2 files changed, 376 insertions(+), 372 deletions(-) diff --git a/python-client/Dockerfile.dev b/python-client/Dockerfile.dev index d45e3ba43..620ae06a7 100644 --- a/python-client/Dockerfile.dev +++ b/python-client/Dockerfile.dev @@ -5,10 +5,6 @@ RUN apt-get update \ && apt-get install -y python3 python3-pip build-essential openjdk-11-jdk \ && apt-get install -y sudo git -RUN pip3 install twine coverage-badge python-terrier coverage - -RUN pip3 install pytest docker - RUN apt-get install -y curl \ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ && chmod a+r /etc/apt/keyrings/docker.gpg \ @@ -29,3 +25,6 @@ RUN useradd -ms /bin/bash dev \ && groupadd -g 973 docker \ && usermod -aG docker dev USER dev + +RUN pip3 install twine coverage-badge python-terrier coverage +RUN pip3 install pytest docker \ No newline at end of file diff --git a/python-client/tests/ir_datasets_test.py b/python-client/tests/ir_datasets_test.py index 74d4ab0df..8af51b7c5 100644 --- a/python-client/tests/ir_datasets_test.py +++ b/python-client/tests/ir_datasets_test.py @@ -1,410 +1,415 @@ from tira.third_party_integrations import register_rerank_data_to_ir_datasets, load_ir_datasets, ensure_pyterrier_is_loaded import ir_datasets import os +import unittest -ensure_pyterrier_is_loaded(patch_ir_datasets=False) -register_rerank_data_to_ir_datasets('tests/sample-tirex-rerank-data', 'tirex-sample-dataset', 'vaswani') +# TODO: this file still uses "unclean" assertions and should be converted to use unittest assertions - -def test_for_existing_document_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - dataset = ir_datasets.load('tirex-sample-dataset') +class TestIRDatasets(unittest.TestCase): - expected_text = "microwave spectroscopy includes chapters on spectroscope technique\nand design on measurements on gases liquids and solids on nuclear\nproperties on molecular structure and on further possible applications\nof microwaves\n" + @classmethod + def setUpClass(cls): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + register_rerank_data_to_ir_datasets('tests/sample-tirex-rerank-data', 'tirex-sample-dataset', 'vaswani') - assert expected_text == dataset.docs_store().get('8172').text - assert '8172' == dataset.docs_store().get('8172').doc_id + def test_for_existing_document_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + dataset = ir_datasets.load('tirex-sample-dataset') + expected_text = "microwave spectroscopy includes chapters on spectroscope technique\nand design on measurements on gases liquids and solids on nuclear\nproperties on molecular structure and on further possible applications\nof microwaves\n" -def test_for_existing_document_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - dataset = ir_datasets.load('tirex-sample-dataset') + assert expected_text == dataset.docs_store().get('8172').text + assert '8172' == dataset.docs_store().get('8172').doc_id - expected_text = "microwave spectroscopy includes chapters on spectroscope technique\nand design on measurements on gases liquids and solids on nuclear\nproperties on molecular structure and on further possible applications\nof microwaves\n" - assert expected_text == dataset.docs_store().get('8172').text - assert '8172' == dataset.docs_store().get('8172').doc_id + def test_for_existing_document_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + dataset = ir_datasets.load('tirex-sample-dataset') + expected_text = "microwave spectroscopy includes chapters on spectroscope technique\nand design on measurements on gases liquids and solids on nuclear\nproperties on molecular structure and on further possible applications\nof microwaves\n" -def test_for_non_existing_document_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - dataset = ir_datasets.load('tirex-sample-dataset') + assert expected_text == dataset.docs_store().get('8172').text + assert '8172' == dataset.docs_store().get('8172').doc_id - assert dataset.docs_store().get('does-not-exist') is None + def test_for_non_existing_document_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + dataset = ir_datasets.load('tirex-sample-dataset') -def test_for_non_existing_document_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - dataset = ir_datasets.load('tirex-sample-dataset') + assert dataset.docs_store().get('does-not-exist') is None - assert dataset.docs_store().get('does-not-exist') is None + def test_for_non_existing_document_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + dataset = ir_datasets.load('tirex-sample-dataset') -def test_for_queries_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - dataset = ir_datasets.load('tirex-sample-dataset') + assert dataset.docs_store().get('does-not-exist') is None - assert len(list(dataset.queries_iter())) == 1 - assert 'MEASUREMENT OF DIELECTRIC CONSTANT OF LIQUIDS BY THE USE OF MICROWAVE TECHNIQUES\n' == list(dataset.queries_iter())[0].text + def test_for_queries_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + dataset = ir_datasets.load('tirex-sample-dataset') -def test_for_queries_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - dataset = ir_datasets.load('tirex-sample-dataset') + assert len(list(dataset.queries_iter())) == 1 + assert 'MEASUREMENT OF DIELECTRIC CONSTANT OF LIQUIDS BY THE USE OF MICROWAVE TECHNIQUES\n' == list(dataset.queries_iter())[0].text - assert len(list(dataset.queries_iter())) == 1 - assert 'MEASUREMENT OF DIELECTRIC CONSTANT OF LIQUIDS BY THE USE OF MICROWAVE TECHNIQUES\n' == list(dataset.queries_iter())[0].text + def test_for_queries_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + dataset = ir_datasets.load('tirex-sample-dataset') -def test_for_scoreddocs_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - dataset = ir_datasets.load('tirex-sample-dataset') - print(list(dataset.scoreddocs)) - assert len(list(dataset.scoreddocs)) == 10 - assert '8172' == list(dataset.scoreddocs)[0].doc_id - assert '1' == list(dataset.scoreddocs)[0].query_id + assert len(list(dataset.queries_iter())) == 1 + assert 'MEASUREMENT OF DIELECTRIC CONSTANT OF LIQUIDS BY THE USE OF MICROWAVE TECHNIQUES\n' == list(dataset.queries_iter())[0].text -def test_for_scoreddocs_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - dataset = ir_datasets.load('tirex-sample-dataset') - print(list(dataset.scoreddocs)) - assert len(list(dataset.scoreddocs)) == 10 - assert '8172' == list(dataset.scoreddocs)[0].doc_id - assert '1' == list(dataset.scoreddocs)[0].query_id + def test_for_scoreddocs_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + dataset = ir_datasets.load('tirex-sample-dataset') + print(list(dataset.scoreddocs)) + assert len(list(dataset.scoreddocs)) == 10 + assert '8172' == list(dataset.scoreddocs)[0].doc_id + assert '1' == list(dataset.scoreddocs)[0].query_id -def test_loading_raw_ir_datasets_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - ir_datasets = load_ir_datasets() - dataset = ir_datasets.load('cranfield') - queries = {i.query_id: i.text for i in dataset.queries_iter()} + def test_for_scoreddocs_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + dataset = ir_datasets.load('tirex-sample-dataset') + print(list(dataset.scoreddocs)) + assert len(list(dataset.scoreddocs)) == 10 + assert '8172' == list(dataset.scoreddocs)[0].doc_id + assert '1' == list(dataset.scoreddocs)[0].query_id - assert len(list(dataset.queries_iter())) == 225 - assert queries['269'] == 'has a criterion been established for determining the axial compressor\nchoking line .' + def test_loading_raw_ir_datasets_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + ir_datasets = load_ir_datasets() + dataset = ir_datasets.load('cranfield') + queries = {i.query_id: i.text for i in dataset.queries_iter()} -def test_loading_raw_ir_datasets_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - ir_datasets = load_ir_datasets() - dataset = ir_datasets.load('cranfield') - queries = {i.query_id: i.text for i in dataset.queries_iter()} + assert len(list(dataset.queries_iter())) == 225 + assert queries['269'] == 'has a criterion been established for determining the axial compressor\nchoking line .' - assert len(list(dataset.queries_iter())) == 225 - assert queries['269'] == 'has a criterion been established for determining the axial compressor\nchoking line .' + def test_loading_raw_ir_datasets_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + ir_datasets = load_ir_datasets() + dataset = ir_datasets.load('cranfield') + queries = {i.query_id: i.text for i in dataset.queries_iter()} -def test_loading_queries_from_ir_datasets_from_custom_directory_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - ir_datasets = load_ir_datasets() - del os.environ['TIRA_INPUT_DATASET'] - dataset = ir_datasets.load('does-not-exist-and-is-not-used') - queries = {i.query_id: i.text for i in dataset.queries_iter()} + assert len(list(dataset.queries_iter())) == 225 + assert queries['269'] == 'has a criterion been established for determining the axial compressor\nchoking line .' - assert len(list(dataset.queries_iter())) == 2 - assert queries['1'] == 'fox jumps above animal' + def test_loading_queries_from_ir_datasets_from_custom_directory_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + ir_datasets = load_ir_datasets() + del os.environ['TIRA_INPUT_DATASET'] + dataset = ir_datasets.load('does-not-exist-and-is-not-used') + queries = {i.query_id: i.text for i in dataset.queries_iter()} -def test_loading_queries_from_ir_datasets_from_custom_directory_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - ir_datasets = load_ir_datasets() - del os.environ['TIRA_INPUT_DATASET'] - dataset = ir_datasets.load('does-not-exist-and-is-not-used') - queries = {i.query_id: i.text for i in dataset.queries_iter()} + assert len(list(dataset.queries_iter())) == 2 + assert queries['1'] == 'fox jumps above animal' - assert len(list(dataset.queries_iter())) == 2 - assert queries['1'] == 'fox jumps above animal' -def test_loading_queries_from_ir_datasets_from_custom_directory_2_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank-without-documents' - ir_datasets = load_ir_datasets() - del os.environ['TIRA_INPUT_DATASET'] - dataset = ir_datasets.load('does-not-exist-and-is-not-used') - queries = {i.query_id: i.text for i in dataset.queries_iter()} - - assert len(list(dataset.queries_iter())) == 2 - assert queries['1'] == 'fox jumps above animal' - -def test_loading_queries_from_ir_datasets_from_custom_directory_2_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank-without-documents' - ir_datasets = load_ir_datasets() - del os.environ['TIRA_INPUT_DATASET'] - dataset = ir_datasets.load('does-not-exist-and-is-not-used') - queries = {i.query_id: i.text for i in dataset.queries_iter()} - - assert len(list(dataset.queries_iter())) == 2 - assert queries['1'] == 'fox jumps above animal' - -def test_loading_documents_from_ir_datasets_from_custom_directory_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - ir_datasets = load_ir_datasets() - del os.environ['TIRA_INPUT_DATASET'] - dataset = ir_datasets.load('does-not-exist-and-is-not-used') - docs = {i.doc_id: i.text for i in dataset.docs_iter()} - - assert len(list(dataset.docs_iter())) == 5 - assert docs['pangram-02'] == 'Quick fox jumps nightly above wizard.' - -def test_loading_documents_from_ir_datasets_from_custom_directory_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - ir_datasets = load_ir_datasets() - del os.environ['TIRA_INPUT_DATASET'] - dataset = ir_datasets.load('does-not-exist-and-is-not-used') - docs = {i.doc_id: i.text for i in dataset.docs_iter()} - - assert len(list(dataset.docs_iter())) == 5 - assert docs['pangram-02'] == 'Quick fox jumps nightly above wizard.' - -def test_loading_docs_store_from_ir_datasets_from_custom_directory_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - ir_datasets = load_ir_datasets() - del os.environ['TIRA_INPUT_DATASET'] - dataset = ir_datasets.load('does-not-exist-and-is-not-used') - - assert dataset.docs_store()['pangram-03'].text == 'The jay, pig, fox, zebra and my wolves quack!' - -def test_loading_docs_store_from_ir_datasets_from_custom_directory_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - ir_datasets = load_ir_datasets() - del os.environ['TIRA_INPUT_DATASET'] - dataset = ir_datasets.load('does-not-exist-and-is-not-used') - - assert dataset.docs_store()['pangram-03'].text == 'The jay, pig, fox, zebra and my wolves quack!' - -def test_no_patching_for_pyterrier_datasets_01(): - ensure_pyterrier_is_loaded() - import pyterrier as pt - dataset = pt.get_dataset('irds:cranfield') - queries = {i['qid']: i['query'] for _, i in dataset.get_topics().iterrows()} - - assert len(dataset.get_topics()) == 225 - assert queries['269'] == 'has a criterion been established for determining the axial compressor choking line' - -def test_no_patching_for_pyterrier_datasets_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - import pyterrier as pt - dataset = pt.get_dataset('irds:cranfield') - queries = {i['qid']: i['query'] for _, i in dataset.get_topics().iterrows()} - - assert len(dataset.get_topics()) == 225 - assert queries['269'] == 'has a criterion been established for determining the axial compressor choking line' - -def test_patching_for_pyterrier_datasets_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - import ir_datasets - l = ir_datasets.load - ensure_pyterrier_is_loaded() - import pyterrier as pt - dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') - ir_datasets.load = l - del os.environ['TIRA_INPUT_DATASET'] - - queries = {i['qid']: i['query'] for _, i in dataset.get_topics().iterrows()} - - assert len(list(dataset.get_topics('title'))) == 2 - assert queries['1'] == 'fox jumps above animal' - -def test_patching_for_pyterrier_datasets_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - import ir_datasets - l = ir_datasets.load - ensure_pyterrier_is_loaded() - import pyterrier as pt - dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') - ir_datasets.load = l - del os.environ['TIRA_INPUT_DATASET'] - - queries = {i['qid']: i['query'] for _, i in dataset.get_topics().iterrows()} - - assert len(list(dataset.get_topics('title'))) == 2 - assert queries['1'] == 'fox jumps above animal' - -def test_patching_for_pyterrier_datasets_for_text_queries_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - import ir_datasets - l = ir_datasets.load - ensure_pyterrier_is_loaded() - import pyterrier as pt - dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') - ir_datasets.load = l - del os.environ['TIRA_INPUT_DATASET'] - - queries = {i['qid']: i['query'] for _, i in dataset.get_topics('text').iterrows()} - - assert len(list(dataset.get_topics('text'))) == 2 - assert queries['1'] == 'fox jumps above animal' - -def test_patching_for_pyterrier_datasets_for_text_queries_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - import ir_datasets - l = ir_datasets.load - ensure_pyterrier_is_loaded() - import pyterrier as pt - dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') - ir_datasets.load = l - del os.environ['TIRA_INPUT_DATASET'] - - queries = {i['qid']: i['query'] for _, i in dataset.get_topics('text').iterrows()} - - assert len(list(dataset.get_topics('text'))) == 2 - assert queries['1'] == 'fox jumps above animal' - -def test_patching_for_pyterrier_datasets_for_title_queries_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - import ir_datasets - l = ir_datasets.load - ensure_pyterrier_is_loaded() - import pyterrier as pt - dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') - ir_datasets.load = l - del os.environ['TIRA_INPUT_DATASET'] - - queries = {i['qid']: i['query'] for _, i in dataset.get_topics('title').iterrows()} - - assert len(list(dataset.get_topics('title'))) == 2 - assert queries['1'] == 'fox jumps above animal' - -def test_patching_for_pyterrier_datasets_for_title_queries_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' - import ir_datasets - l = ir_datasets.load - ensure_pyterrier_is_loaded() - import pyterrier as pt - dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') - ir_datasets.load = l - del os.environ['TIRA_INPUT_DATASET'] - - queries = {i['qid']: i['query'] for _, i in dataset.get_topics('title').iterrows()} - - assert len(list(dataset.get_topics('title'))) == 2 - assert queries['1'] == 'fox jumps above animal' - -def test_loading_ir_dataset_via_rest_api_from_tira_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - from tira.third_party_integrations import ir_datasets - dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') - - assert dataset.has_docs(), "dataset has no documents" - assert dataset.has_queries(), "dataset has no queries" - assert dataset.has_qrels(), "dataset has no qrels" - -def test_loading_ir_dataset_via_rest_api_from_tira_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - from tira.third_party_integrations import ir_datasets - dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') - - assert dataset.has_docs(), "dataset has no documents" - assert dataset.has_queries(), "dataset has no queries" - assert dataset.has_qrels(), "dataset has no qrels" - -def test_loading_qrels_via_rest_api_from_tira_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - from tira.third_party_integrations import ir_datasets - dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') - - assert len([i for i in dataset.qrels_iter()]) == 2635 - assert '''TrecQrel(query_id='1', doc_id='2005.ipm_journal-ir0anthology0volumeA41A1.7', relevance=1, iteration='0')''' == str([i for i in dataset.qrels_iter()][0]) - -def test_loading_qrels_via_rest_api_from_tira_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - from tira.third_party_integrations import ir_datasets - dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') - - assert len([i for i in dataset.qrels_iter()]) == 2635 - assert '''TrecQrel(query_id='1', doc_id='2005.ipm_journal-ir0anthology0volumeA41A1.7', relevance=1, iteration='0')''' == str([i for i in dataset.qrels_iter()][0]) - -def test_loading_docs_via_rest_api_from_tira_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - from tira.third_party_integrations import ir_datasets - dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') - - assert len([i for i in dataset.docs_iter()]) == 53673 - docsstore = dataset.docs_store() - - assert docsstore.get('2005.ipm_journal-ir0anthology0volumeA41A1.7').text.startswith('A probabilistic model for stemmer generation AbstractIn this paper') - -def test_loading_docs_via_rest_api_from_tira_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - from tira.third_party_integrations import ir_datasets - dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') - - assert len([i for i in dataset.docs_iter()]) == 53673 - docsstore = dataset.docs_store() - - assert docsstore.get('2005.ipm_journal-ir0anthology0volumeA41A1.7').text.startswith('A probabilistic model for stemmer generation AbstractIn this paper') - -def test_loading_topics_via_rest_api_from_tira_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - from tira.third_party_integrations import ir_datasets - dataset = ir_datasets.topics_file('ir-lab-jena-leipzig-wise-2023/training-20231104-training') - - assert dataset.endswith('/training-20231104-training/truth-data/queries.xml') - -def test_loading_topics_via_rest_api_from_tira_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - from tira.third_party_integrations import ir_datasets - dataset = ir_datasets.topics_file('ir-lab-jena-leipzig-wise-2023/training-20231104-training') - - assert dataset.endswith('/training-20231104-training/truth-data/queries.xml') - -def test_loading_topics_via_rest_api_from_rerank_dataset_from_tira_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/re-ranking-outputs' - ir_datasets = load_ir_datasets() - dataset = ir_datasets.load('workshop-on-open-web-search/re-ranking-20231027-training') - del os.environ['TIRA_INPUT_DATASET'] - - assert len([i for i in dataset.queries_iter()]) == 2 - assert len([i for i in dataset.docs_iter()]) == 3 - -def test_loading_queries_via_rest_api_from_tira_01(): - ensure_pyterrier_is_loaded(patch_ir_datasets=False) - from tira.third_party_integrations import ir_datasets - dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') - - assert 68 == len(list(dataset.queries_iter())) - print(str(list(dataset.queries_iter())[0])) - assert '''TirexQuery(query_id='1', text='retrieval system improving effectiveness', title='retrieval system improving effectiveness', query='retrieval system improving effectiveness', description='What papers focus on improving the effectiveness of a retrieval system?', narrative='Relevant papers include research on what makes a retrieval system effective and what improves the effectiveness of a retrieval system. Papers that focus on improving something else or improving the effectiveness of a system that is not a retrieval system are not relevant.')''' == str(list(dataset.queries_iter())[0]) - -def test_loading_queries_via_rest_api_from_tira_02(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - from tira.third_party_integrations import ir_datasets - dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') - - assert 68 == len(list(dataset.queries_iter())) - print(str(list(dataset.queries_iter())[0])) - assert '''TirexQuery(query_id='1', text='retrieval system improving effectiveness', title='retrieval system improving effectiveness', query='retrieval system improving effectiveness', description='What papers focus on improving the effectiveness of a retrieval system?', narrative='Relevant papers include research on what makes a retrieval system effective and what improves the effectiveness of a retrieval system. Papers that focus on improving something else or improving the effectiveness of a system that is not a retrieval system are not relevant.')''' == str(list(dataset.queries_iter())[0]) - -def test_patching_for_pyterrier_datasets_to_tira(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - import pyterrier as pt - dataset = pt.get_dataset('irds:workshop-on-open-web-search/retrieval-20231027-training') - queries = {i['qid']: i['title'] for _, i in dataset.get_topics().iterrows()} - - assert len(dataset.get_topics()) == 3 - assert queries['1'] == 'hubble telescope achievements' - -def test_patching_for_pyterrier_datasets_with_qrels_to_tira(): - ensure_pyterrier_is_loaded(patch_ir_datasets=True) - import pyterrier as pt - dataset = pt.get_dataset('irds:workshop-on-open-web-search/retrieval-20231027-training') - qrels =[i.to_dict() for _, i in dataset.get_qrels().iterrows()] - - assert len(dataset.get_qrels()) == 3 - assert qrels[0] == {'docno': 'doc-1', 'iteration': '0', 'label': 1, 'qid': '1'} + def test_loading_queries_from_ir_datasets_from_custom_directory_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + ir_datasets = load_ir_datasets() + del os.environ['TIRA_INPUT_DATASET'] + dataset = ir_datasets.load('does-not-exist-and-is-not-used') + queries = {i.query_id: i.text for i in dataset.queries_iter()} + assert len(list(dataset.queries_iter())) == 2 + assert queries['1'] == 'fox jumps above animal' + + def test_loading_queries_from_ir_datasets_from_custom_directory_2_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank-without-documents' + ir_datasets = load_ir_datasets() + del os.environ['TIRA_INPUT_DATASET'] + dataset = ir_datasets.load('does-not-exist-and-is-not-used') + queries = {i.query_id: i.text for i in dataset.queries_iter()} + + assert len(list(dataset.queries_iter())) == 2 + assert queries['1'] == 'fox jumps above animal' + + def test_loading_queries_from_ir_datasets_from_custom_directory_2_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank-without-documents' + ir_datasets = load_ir_datasets() + del os.environ['TIRA_INPUT_DATASET'] + dataset = ir_datasets.load('does-not-exist-and-is-not-used') + queries = {i.query_id: i.text for i in dataset.queries_iter()} + + assert len(list(dataset.queries_iter())) == 2 + assert queries['1'] == 'fox jumps above animal' + + def test_loading_documents_from_ir_datasets_from_custom_directory_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + ir_datasets = load_ir_datasets() + del os.environ['TIRA_INPUT_DATASET'] + dataset = ir_datasets.load('does-not-exist-and-is-not-used') + docs = {i.doc_id: i.text for i in dataset.docs_iter()} + + assert len(list(dataset.docs_iter())) == 5 + assert docs['pangram-02'] == 'Quick fox jumps nightly above wizard.' + + def test_loading_documents_from_ir_datasets_from_custom_directory_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + ir_datasets = load_ir_datasets() + del os.environ['TIRA_INPUT_DATASET'] + dataset = ir_datasets.load('does-not-exist-and-is-not-used') + docs = {i.doc_id: i.text for i in dataset.docs_iter()} + + assert len(list(dataset.docs_iter())) == 5 + assert docs['pangram-02'] == 'Quick fox jumps nightly above wizard.' + + def test_loading_docs_store_from_ir_datasets_from_custom_directory_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + ir_datasets = load_ir_datasets() + del os.environ['TIRA_INPUT_DATASET'] + dataset = ir_datasets.load('does-not-exist-and-is-not-used') + + assert dataset.docs_store()['pangram-03'].text == 'The jay, pig, fox, zebra and my wolves quack!' + + def test_loading_docs_store_from_ir_datasets_from_custom_directory_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + ir_datasets = load_ir_datasets() + del os.environ['TIRA_INPUT_DATASET'] + dataset = ir_datasets.load('does-not-exist-and-is-not-used') + + assert dataset.docs_store()['pangram-03'].text == 'The jay, pig, fox, zebra and my wolves quack!' + + def test_no_patching_for_pyterrier_datasets_01(self): + ensure_pyterrier_is_loaded() + import pyterrier as pt + dataset = pt.get_dataset('irds:cranfield') + queries = {i['qid']: i['query'] for _, i in dataset.get_topics().iterrows()} + + assert len(dataset.get_topics()) == 225 + assert queries['269'] == 'has a criterion been established for determining the axial compressor choking line' + + def test_no_patching_for_pyterrier_datasets_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + import pyterrier as pt + dataset = pt.get_dataset('irds:cranfield') + queries = {i['qid']: i['query'] for _, i in dataset.get_topics().iterrows()} + + assert len(dataset.get_topics()) == 225 + assert queries['269'] == 'has a criterion been established for determining the axial compressor choking line' + + def test_patching_for_pyterrier_datasets_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + import ir_datasets + l = ir_datasets.load + ensure_pyterrier_is_loaded() + import pyterrier as pt + dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') + ir_datasets.load = l + del os.environ['TIRA_INPUT_DATASET'] + + queries = {i['qid']: i['query'] for _, i in dataset.get_topics().iterrows()} + + assert len(list(dataset.get_topics('title'))) == 2 + assert queries['1'] == 'fox jumps above animal' + + def test_patching_for_pyterrier_datasets_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + import ir_datasets + l = ir_datasets.load + ensure_pyterrier_is_loaded() + import pyterrier as pt + dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') + ir_datasets.load = l + del os.environ['TIRA_INPUT_DATASET'] + + queries = {i['qid']: i['query'] for _, i in dataset.get_topics().iterrows()} + + assert len(list(dataset.get_topics('title'))) == 2 + assert queries['1'] == 'fox jumps above animal' + + def test_patching_for_pyterrier_datasets_for_text_queries_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + import ir_datasets + l = ir_datasets.load + ensure_pyterrier_is_loaded() + import pyterrier as pt + dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') + ir_datasets.load = l + del os.environ['TIRA_INPUT_DATASET'] + + queries = {i['qid']: i['query'] for _, i in dataset.get_topics('text').iterrows()} + + assert len(list(dataset.get_topics('text'))) == 2 + assert queries['1'] == 'fox jumps above animal' + + def test_patching_for_pyterrier_datasets_for_text_queries_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + import ir_datasets + l = ir_datasets.load + ensure_pyterrier_is_loaded() + import pyterrier as pt + dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') + ir_datasets.load = l + del os.environ['TIRA_INPUT_DATASET'] + + queries = {i['qid']: i['query'] for _, i in dataset.get_topics('text').iterrows()} + + assert len(list(dataset.get_topics('text'))) == 2 + assert queries['1'] == 'fox jumps above animal' + + def test_patching_for_pyterrier_datasets_for_title_queries_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + import ir_datasets + l = ir_datasets.load + ensure_pyterrier_is_loaded() + import pyterrier as pt + dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') + ir_datasets.load = l + del os.environ['TIRA_INPUT_DATASET'] + + queries = {i['qid']: i['query'] for _, i in dataset.get_topics('title').iterrows()} + + assert len(list(dataset.get_topics('title'))) == 2 + assert queries['1'] == 'fox jumps above animal' + + def test_patching_for_pyterrier_datasets_for_title_queries_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/sample-input-full-rank' + import ir_datasets + l = ir_datasets.load + ensure_pyterrier_is_loaded() + import pyterrier as pt + dataset = pt.get_dataset('irds:does-not-exist-and-is-not-used') + ir_datasets.load = l + del os.environ['TIRA_INPUT_DATASET'] + + queries = {i['qid']: i['query'] for _, i in dataset.get_topics('title').iterrows()} + + assert len(list(dataset.get_topics('title'))) == 2 + assert queries['1'] == 'fox jumps above animal' + + def test_loading_ir_dataset_via_rest_api_from_tira_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + from tira.third_party_integrations import ir_datasets + dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') + + assert dataset.has_docs(), "dataset has no documents" + assert dataset.has_queries(), "dataset has no queries" + assert dataset.has_qrels(), "dataset has no qrels" + + def test_loading_ir_dataset_via_rest_api_from_tira_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + from tira.third_party_integrations import ir_datasets + dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') + + assert dataset.has_docs(), "dataset has no documents" + assert dataset.has_queries(), "dataset has no queries" + assert dataset.has_qrels(), "dataset has no qrels" + + def test_loading_qrels_via_rest_api_from_tira_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + from tira.third_party_integrations import ir_datasets + dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') + + assert len([i for i in dataset.qrels_iter()]) == 2635 + assert '''TrecQrel(query_id='1', doc_id='2005.ipm_journal-ir0anthology0volumeA41A1.7', relevance=1, iteration='0')''' == str([i for i in dataset.qrels_iter()][0]) + + def test_loading_qrels_via_rest_api_from_tira_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + from tira.third_party_integrations import ir_datasets + dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') + + assert len([i for i in dataset.qrels_iter()]) == 2635 + assert '''TrecQrel(query_id='1', doc_id='2005.ipm_journal-ir0anthology0volumeA41A1.7', relevance=1, iteration='0')''' == str([i for i in dataset.qrels_iter()][0]) + + def test_loading_docs_via_rest_api_from_tira_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + from tira.third_party_integrations import ir_datasets + dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') + + assert len([i for i in dataset.docs_iter()]) == 53673 + docsstore = dataset.docs_store() + + assert docsstore.get('2005.ipm_journal-ir0anthology0volumeA41A1.7').text.startswith('A probabilistic model for stemmer generation AbstractIn this paper') + + def test_loading_docs_via_rest_api_from_tira_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + from tira.third_party_integrations import ir_datasets + dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') + + assert len([i for i in dataset.docs_iter()]) == 53673 + docsstore = dataset.docs_store() + + assert docsstore.get('2005.ipm_journal-ir0anthology0volumeA41A1.7').text.startswith('A probabilistic model for stemmer generation AbstractIn this paper') + + def test_loading_topics_via_rest_api_from_tira_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + from tira.third_party_integrations import ir_datasets + dataset = ir_datasets.topics_file('ir-lab-jena-leipzig-wise-2023/training-20231104-training') + + assert dataset.endswith('/training-20231104-training/truth-data/queries.xml') + + def test_loading_topics_via_rest_api_from_tira_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + from tira.third_party_integrations import ir_datasets + dataset = ir_datasets.topics_file('ir-lab-jena-leipzig-wise-2023/training-20231104-training') + + assert dataset.endswith('/training-20231104-training/truth-data/queries.xml') + + def test_loading_topics_via_rest_api_from_rerank_dataset_from_tira_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + os.environ['TIRA_INPUT_DATASET'] = 'tests/resources/re-ranking-outputs' + ir_datasets = load_ir_datasets() + dataset = ir_datasets.load('workshop-on-open-web-search/re-ranking-20231027-training') + del os.environ['TIRA_INPUT_DATASET'] + + assert len([i for i in dataset.queries_iter()]) == 2 + assert len([i for i in dataset.docs_iter()]) == 3 + + def test_loading_queries_via_rest_api_from_tira_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + from tira.third_party_integrations import ir_datasets + dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') + + assert 68 == len(list(dataset.queries_iter())) + print(str(list(dataset.queries_iter())[0])) + assert '''TirexQuery(query_id='1', text='retrieval system improving effectiveness', title='retrieval system improving effectiveness', query='retrieval system improving effectiveness', description='What papers focus on improving the effectiveness of a retrieval system?', narrative='Relevant papers include research on what makes a retrieval system effective and what improves the effectiveness of a retrieval system. Papers that focus on improving something else or improving the effectiveness of a system that is not a retrieval system are not relevant.')''' == str(list(dataset.queries_iter())[0]) + + def test_loading_queries_via_rest_api_from_tira_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + from tira.third_party_integrations import ir_datasets + dataset = ir_datasets.load('ir-lab-jena-leipzig-sose-2023/iranthology-20230618-training') + + assert 68 == len(list(dataset.queries_iter())) + print(str(list(dataset.queries_iter())[0])) + assert '''TirexQuery(query_id='1', text='retrieval system improving effectiveness', title='retrieval system improving effectiveness', query='retrieval system improving effectiveness', description='What papers focus on improving the effectiveness of a retrieval system?', narrative='Relevant papers include research on what makes a retrieval system effective and what improves the effectiveness of a retrieval system. Papers that focus on improving something else or improving the effectiveness of a system that is not a retrieval system are not relevant.')''' == str(list(dataset.queries_iter())[0]) + + def test_patching_for_pyterrier_datasets_to_tira(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + import pyterrier as pt + dataset = pt.get_dataset('irds:workshop-on-open-web-search/retrieval-20231027-training') + queries = {i['qid']: i['title'] for _, i in dataset.get_topics().iterrows()} + + assert len(dataset.get_topics()) == 3 + assert queries['1'] == 'hubble telescope achievements' + + def test_patching_for_pyterrier_datasets_with_qrels_to_tira(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + import pyterrier as pt + dataset = pt.get_dataset('irds:workshop-on-open-web-search/retrieval-20231027-training') + qrels =[i.to_dict() for _, i in dataset.get_qrels().iterrows()] + + assert len(dataset.get_qrels()) == 3 + assert qrels[0] == {'docno': 'doc-1', 'iteration': '0', 'label': 1, 'qid': '1'} From 61cdd92fc1aca8d0b31466c1c3cf891055a4e7ac Mon Sep 17 00:00:00 2001 From: Tim Hagen Date: Tue, 21 Nov 2023 12:32:22 +0000 Subject: [PATCH 05/18] added support for scoreddocs_count --- python-client/tests/ir_datasets_test.py | 12 ++++++++++++ python-client/tira/ir_datasets_util.py | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/python-client/tests/ir_datasets_test.py b/python-client/tests/ir_datasets_test.py index 8af51b7c5..2a0f6d4a6 100644 --- a/python-client/tests/ir_datasets_test.py +++ b/python-client/tests/ir_datasets_test.py @@ -81,6 +81,18 @@ def test_for_scoreddocs_02(self): assert '1' == list(dataset.scoreddocs)[0].query_id + def test_for_scoreddocs_count_01(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=False) + dataset = ir_datasets.load('tirex-sample-dataset') + self.assertEqual(dataset.scoreddocs_count(), 10) + + + def test_for_scoreddocs_count_02(self): + ensure_pyterrier_is_loaded(patch_ir_datasets=True) + dataset = ir_datasets.load('tirex-sample-dataset') + self.assertEqual(dataset.scoreddocs_count(), 10) + + def test_loading_raw_ir_datasets_01(self): ensure_pyterrier_is_loaded(patch_ir_datasets=False) ir_datasets = load_ir_datasets() diff --git a/python-client/tira/ir_datasets_util.py b/python-client/tira/ir_datasets_util.py index 1292786b8..14ecf5462 100644 --- a/python-client/tira/ir_datasets_util.py +++ b/python-client/tira/ir_datasets_util.py @@ -42,6 +42,7 @@ def register_dataset_from_re_rank_file(ir_dataset_id, df_re_rank, original_ir_da scoreddocs = __scored_docs(df_re_rank, original_dataset) dataset = Dataset(docs, queries, qrels, scoreddocs) + dataset.metadata = None ir_datasets.registry.register(ir_dataset_id, dataset) __check_registration_was_successful(ir_dataset_id, original_ir_datasets_id is None) @@ -189,6 +190,9 @@ def __init__(self, docs): def scoreddocs_iter(self): return deepcopy(self.docs).__iter__() + def scoreddocs_count(self) -> int: + return len(docs) + docs = [] for i in stream_all_lines(input_file, True): From 052dd96d327572bc1de1655b5ae5b79150258b63 Mon Sep 17 00:00:00 2001 From: Tim Hagen Date: Tue, 21 Nov 2023 14:01:06 +0000 Subject: [PATCH 06/18] Added basic sphinx integration --- python-client/.gitignore | 6 ++++ python-client/Dockerfile.dev | 10 +++++-- python-client/sphinx/Makefile | 20 +++++++++++++ python-client/sphinx/conf.py | 53 ++++++++++++++++++++++++++++++++++ python-client/sphinx/index.rst | 18 ++++++++++++ python-client/sphinx/make.bat | 35 ++++++++++++++++++++++ 6 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 python-client/sphinx/Makefile create mode 100644 python-client/sphinx/conf.py create mode 100644 python-client/sphinx/index.rst create mode 100644 python-client/sphinx/make.bat diff --git a/python-client/.gitignore b/python-client/.gitignore index 567609b12..a34749379 100644 --- a/python-client/.gitignore +++ b/python-client/.gitignore @@ -1 +1,7 @@ build/ + +.devcontainer/ + +sphinx/_build +sphinx/**.rst +!sphinx/index.rst \ No newline at end of file diff --git a/python-client/Dockerfile.dev b/python-client/Dockerfile.dev index 620ae06a7..e300302b7 100644 --- a/python-client/Dockerfile.dev +++ b/python-client/Dockerfile.dev @@ -1,9 +1,14 @@ #docker build -t webis/tira:python-client-dev-0.0.3 -f Dockerfile.dev . FROM ubuntu:latest +ENV TZ=Europe/Berlin +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + RUN apt-get update \ && apt-get install -y python3 python3-pip build-essential openjdk-11-jdk \ - && apt-get install -y sudo git + && apt-get install -y sudo git locales + +RUN echo "en_US.UTF-8 UTF-8" | sudo tee -a /etc/locale.gen && locale-gen RUN apt-get install -y curl \ && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \ @@ -27,4 +32,5 @@ RUN useradd -ms /bin/bash dev \ USER dev RUN pip3 install twine coverage-badge python-terrier coverage -RUN pip3 install pytest docker \ No newline at end of file +RUN pip3 install pytest docker +RUN pip3 install sphinx furo \ No newline at end of file diff --git a/python-client/sphinx/Makefile b/python-client/sphinx/Makefile new file mode 100644 index 000000000..d4bb2cbb9 --- /dev/null +++ b/python-client/sphinx/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/python-client/sphinx/conf.py b/python-client/sphinx/conf.py new file mode 100644 index 000000000..f94942b54 --- /dev/null +++ b/python-client/sphinx/conf.py @@ -0,0 +1,53 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../tira/')) + + +# -- Project information ----------------------------------------------------- + +project = 'TIRA' +copyright = '2023, Webis' +author = 'Webis' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'furo' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] \ No newline at end of file diff --git a/python-client/sphinx/index.rst b/python-client/sphinx/index.rst new file mode 100644 index 000000000..c0262cf9d --- /dev/null +++ b/python-client/sphinx/index.rst @@ -0,0 +1,18 @@ +.. TIRA documentation master file, created by + sphinx-quickstart on Tue Nov 21 14:15:16 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to TIRA's documentation! +================================ + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/python-client/sphinx/make.bat b/python-client/sphinx/make.bat new file mode 100644 index 000000000..153be5e2f --- /dev/null +++ b/python-client/sphinx/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd From 2207ed44f75e450b23c91e2da3766ee92f6f93a4 Mon Sep 17 00:00:00 2001 From: Tim Hagen Date: Tue, 21 Nov 2023 15:14:00 +0100 Subject: [PATCH 07/18] Added Actions integration for deploying sphinx documentation --- .github/workflows/build-documentation.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/build-documentation.yml diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation.yml new file mode 100644 index 000000000..9f2714e77 --- /dev/null +++ b/.github/workflows/build-documentation.yml @@ -0,0 +1,23 @@ +name: Documentation +on: + push: + branches: + - development + +jobs: + build-doc: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install Dependencies + run: | + apt-get install python-sphinx furo + cd python-client/sphinx + sphinx-apidoc -o . ../tira + make html + - name: Deploy + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: python-client/sphinx/_build/html + target-folder: nightly From a37a05e578bc9afd82c03f81129ebed3f6c5f967 Mon Sep 17 00:00:00 2001 From: Tim Hagen Date: Tue, 21 Nov 2023 15:17:09 +0100 Subject: [PATCH 08/18] lets hope --- .github/workflows/build-documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation.yml index 9f2714e77..87162edb3 100644 --- a/.github/workflows/build-documentation.yml +++ b/.github/workflows/build-documentation.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v3 - name: Install Dependencies run: | - apt-get install python-sphinx furo + sudo apt-get install -y python-sphinx furo cd python-client/sphinx sphinx-apidoc -o . ../tira make html From e3dc96f28d3ad143fb1b2bbce17ba0a48f1ec7cb Mon Sep 17 00:00:00 2001 From: Tim Hagen Date: Tue, 21 Nov 2023 15:18:08 +0100 Subject: [PATCH 09/18] please --- .github/workflows/build-documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation.yml index 87162edb3..ab939445b 100644 --- a/.github/workflows/build-documentation.yml +++ b/.github/workflows/build-documentation.yml @@ -12,7 +12,7 @@ jobs: uses: actions/checkout@v3 - name: Install Dependencies run: | - sudo apt-get install -y python-sphinx furo + sudo apt-get install -y python3-sphinx furo cd python-client/sphinx sphinx-apidoc -o . ../tira make html From a62a5878ff5762cdd5ae8d8b78c4863c95530c0b Mon Sep 17 00:00:00 2001 From: Tim Hagen Date: Tue, 21 Nov 2023 15:19:28 +0100 Subject: [PATCH 10/18] now though --- .github/workflows/build-documentation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation.yml index ab939445b..f383f38a7 100644 --- a/.github/workflows/build-documentation.yml +++ b/.github/workflows/build-documentation.yml @@ -12,7 +12,8 @@ jobs: uses: actions/checkout@v3 - name: Install Dependencies run: | - sudo apt-get install -y python3-sphinx furo + sudo apt-get install -y python3-pip + pip install sphinx furo cd python-client/sphinx sphinx-apidoc -o . ../tira make html From 979968bc5eaa14ae36cf2ed466e6e2d3fed18d45 Mon Sep 17 00:00:00 2001 From: Tim Hagen Date: Tue, 21 Nov 2023 15:56:24 +0100 Subject: [PATCH 11/18] removed jekyll --- .github/workflows/build-documentation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation.yml index f383f38a7..b5f676103 100644 --- a/.github/workflows/build-documentation.yml +++ b/.github/workflows/build-documentation.yml @@ -17,6 +17,7 @@ jobs: cd python-client/sphinx sphinx-apidoc -o . ../tira make html + touch python-client/sphinx/_build/html/.nojekyll - name: Deploy uses: JamesIves/github-pages-deploy-action@v4 with: From 26168d8f73bae7179460d161f178f5b14e703e8c Mon Sep 17 00:00:00 2001 From: Tim Hagen Date: Tue, 21 Nov 2023 15:57:39 +0100 Subject: [PATCH 12/18] oops --- .github/workflows/build-documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation.yml index b5f676103..6953513b3 100644 --- a/.github/workflows/build-documentation.yml +++ b/.github/workflows/build-documentation.yml @@ -17,7 +17,7 @@ jobs: cd python-client/sphinx sphinx-apidoc -o . ../tira make html - touch python-client/sphinx/_build/html/.nojekyll + touch _build/html/.nojekyll - name: Deploy uses: JamesIves/github-pages-deploy-action@v4 with: From d66ab20e6e6c345d994c37759fbe247d3a808362 Mon Sep 17 00:00:00 2001 From: Tim Hagen Date: Tue, 21 Nov 2023 16:06:57 +0100 Subject: [PATCH 13/18] added support for production and nightly documentation --- ...on.yml => build-documentation-nightly.yml} | 0 .../build-documentation-production.yml | 27 +++++++++++++++++++ 2 files changed, 27 insertions(+) rename .github/workflows/{build-documentation.yml => build-documentation-nightly.yml} (100%) create mode 100644 .github/workflows/build-documentation-production.yml diff --git a/.github/workflows/build-documentation.yml b/.github/workflows/build-documentation-nightly.yml similarity index 100% rename from .github/workflows/build-documentation.yml rename to .github/workflows/build-documentation-nightly.yml diff --git a/.github/workflows/build-documentation-production.yml b/.github/workflows/build-documentation-production.yml new file mode 100644 index 000000000..2cbad31e6 --- /dev/null +++ b/.github/workflows/build-documentation-production.yml @@ -0,0 +1,27 @@ +name: Documentation +on: + push: + branches: + - main + +jobs: + build-doc: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install Dependencies + run: | + sudo apt-get install -y python3-pip + pip install sphinx furo + cd python-client/sphinx + sphinx-apidoc -o . ../tira + make html + touch _build/html/.nojekyll + - name: Deploy + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: python-client/sphinx/_build/html + target-folder: ./ + clean-exclude: + - nightly From 61f47a74749f785a6ea44b3cd855bfe1049ad979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Tue, 21 Nov 2023 18:19:35 +0100 Subject: [PATCH 14/18] Fix diffir integration --- application/src/tira/endpoints/diffir_api.py | 11 ++--- application/src/tira/endpoints/serp_api.py | 40 +------------------ application/test/diffir_tests/__init__.py | 0 .../test/diffir_tests/test_diffir_endpoint.py | 37 +++++++++++++++++ application/test/utils_for_testing.py | 19 +++++++++ 5 files changed, 61 insertions(+), 46 deletions(-) create mode 100644 application/test/diffir_tests/__init__.py create mode 100644 application/test/diffir_tests/test_diffir_endpoint.py diff --git a/application/src/tira/endpoints/diffir_api.py b/application/src/tira/endpoints/diffir_api.py index e28828a07..1bfb63a24 100644 --- a/application/src/tira/endpoints/diffir_api.py +++ b/application/src/tira/endpoints/diffir_api.py @@ -15,13 +15,10 @@ def doc_file_for_run(vm_id, dataset_id, task_id, run_id): for evaluation in model.get_evaluations_of_run(vm_id, run_id): - p = Path(settings.TIRA_ROOT) / "data" / "runs" / dataset_id / vm_id / evaluation / 'output' /\ - ".data-top-10-for-rendering.jsonl" - print(p) - - if p.is_file(): - return p - + for f in [".data-top-10-for-rendering.jsonl.gz", ".data-top-10-for-rendering.jsonl"]: + p = Path(settings.TIRA_ROOT) / "data" / "runs" / dataset_id / vm_id / evaluation / 'output' / f + if p.is_file(): + return p raise ValueError('.data-top-10-for-rendering.jsonl') diff --git a/application/src/tira/endpoints/serp_api.py b/application/src/tira/endpoints/serp_api.py index d82b4848f..5eea99e54 100644 --- a/application/src/tira/endpoints/serp_api.py +++ b/application/src/tira/endpoints/serp_api.py @@ -17,28 +17,16 @@ @add_context @check_permissions def serp(request, context, vm_id, dataset_id, task_id, run_id, topk): - from tira.util import run_cmd_as_documented_background_process - - - import tira.model as modeldb #podman --storage-opt mount_program=/usr/bin/fuse-overlayfs run -v /mnt/ceph:/mnt/ceph:ro -ti webis/tira-application:0.0.45-diffir diffir --dataset cranfield /mnt/ceph/tira/data/runs/cranfield-20230107-training/tira-ir-starter/2023-02-13-12-40-07/output/run.txt if request.method == 'GET': try: - run = model.get_run(dataset_id=None, vm_id=None, run_id=run_id) run_file = Path(settings.TIRA_ROOT) / "data" / "runs" / dataset_id / \ vm_id / run_id / 'output' / 'run.txt' - serp_file = Path(settings.TIRA_ROOT) / "state" / "serp" / "version-0.0.1" / "runs" / dataset_id / vm_id / run_id / "serp.html" - serp_dir = (serp_file / "..").resolve() - run_dir = (run_file / "..").resolve() - Path.mkdir(serp_dir, parents=True, exist_ok=True) if not run_file.is_file(): raise ValueError(f'Error: The expected file {run_file} does not exist.') - if serp_file.is_file(): - return HttpResponse(open(serp_file).read()) - try: from diffir.run import diff_from_local_data except: @@ -49,33 +37,7 @@ def serp(request, context, vm_id, dataset_id, task_id, run_id, topk): _, rendered_serp = diff_from_local_data([str(run_file.resolve())], [str(doc_file)], cli=False, web=True, print_html=False, topk=topk) - serp_dir.mkdir(parents=True, exist_ok=True) - with open(serp_file, 'w') as f: - f.write(rendered_serp) - return HttpResponse(rendered_serp) - - irds_id = load_irds_metadata_of_task(task_id, run['dataset'])['ir_datasets_id'] - image = model.get_dataset(run['dataset'])['irds_docker_image'] - command = [ - ['sudo', 'podman', '--storage-opt', 'mount_program=/usr/bin/fuse-overlayfs', 'run', - '-v', f'{run_dir}:/run_dir:ro', - '-v', f'{serp_dir}:/output-tira-tmp/', - '--entrypoint', 'sh', image, '-c', f'diffir --dataset {irds_id} --web /run_dir/run.txt > /tmp/run.html && mv /tmp/run.html /output-tira-tmp/serp.html'] - ] - - descriptions = ['### RUN diffir'] - - process_id = run_cmd_as_documented_background_process(cmd=command, vm_id=vm_id, task_id=task_id, - title=f'Render SERP for run {run_id}', - descriptions=descriptions) - - while True: - time.sleep(2) - running_process = modeldb.BackendProcess.objects.get(id=process_id) - if running_process.exit_code is not None: - return HttpResponse(open(serp_file).read()) except Exception as e: logger.exception(e) - logger.exception(e) - return JsonResponse({"status": "0", "message": f"Encountered an exception: {e}"}) + return JsonResponse({"status": "1", "message": f"Encountered an exception: {e}"}) diff --git a/application/test/diffir_tests/__init__.py b/application/test/diffir_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/application/test/diffir_tests/test_diffir_endpoint.py b/application/test/diffir_tests/test_diffir_endpoint.py new file mode 100644 index 000000000..c369e56d7 --- /dev/null +++ b/application/test/diffir_tests/test_diffir_endpoint.py @@ -0,0 +1,37 @@ +from django.test import TestCase +from api_access_matrix import ADMIN +from utils_for_testing import dataset_1, dataset_2, dataset_meta, method_for_url_pattern, mock_request, set_up_tira_environment + +url = 'serp//user//dataset///' +diffir = method_for_url_pattern(url) + + +class TestDiffirEndpoint(TestCase): + @classmethod + def setUpClass(cls): + set_up_tira_environment() + + def test_diffir_with_json(self): + # Arrange + request = mock_request(ADMIN, url) + + # Act + actual = diffir(request, vm_id='example_participant', dataset_id='dataset-1', task_id='t1', run_id='run-3-example_participant', topk=10) + + # Assert + self.assertTrue('dsa', actual.content.decode('utf-8').startswith('')) + + def test_diffir_with_json_gz(self): + # Arrange + request = mock_request(ADMIN, url) + + # Act + actual = diffir(request, vm_id='example_participant', dataset_id='dataset-1', task_id='t1', run_id='run-5-example_participant', topk=10) + + # Assert + self.assertTrue('dsa', actual.content.decode('utf-8').startswith('')) + + @classmethod + def tearDownClass(cls): + pass + diff --git a/application/test/utils_for_testing.py b/application/test/utils_for_testing.py index 1080c0b99..69d649250 100644 --- a/application/test/utils_for_testing.py +++ b/application/test/utils_for_testing.py @@ -9,6 +9,7 @@ from datetime import datetime import json from tira.tira_model import model as tira_model +import gzip #Used for some tests now = datetime.now().strftime("%Y%m%d") @@ -153,6 +154,24 @@ def set_up_tira_environment(): k_1 -= 0.1 k_2 += 0.1 + Path('tira-root/data/runs/dataset-1/example_participant/run-3-example_participant/output').mkdir(parents=True, exist_ok=True) + open('tira-root/data/runs/dataset-1/example_participant/run-3-example_participant/output/run.txt', 'w').write( + 'q072210025 0 doc072207504499 1 16.214817060525405 pl2-baseline\nq072210025 0 doc072212607743 2 14.878122569655583 pl2-baseline') + Path('tira-root/data/runs/dataset-1/example_participant/run-3-example_participant-eval/output').mkdir(parents=True, + exist_ok=True) + open('tira-root/data/runs/dataset-1/example_participant/run-3-example_participant-eval/output/.data-top-10-for-rendering.jsonl', 'w').write( + '{"queries": {"q072210025": {"qid": "q072210025","query": "recipe spring roll","original_query": {"query_id": "q072210025","text": "recipe spring roll"}}}, "documents": {}, "qrels": {}}') + + Path('tira-root/data/runs/dataset-1/example_participant/run-5-example_participant/output').mkdir(parents=True, + exist_ok=True) + open('tira-root/data/runs/dataset-1/example_participant/run-5-example_participant/output/run.txt', 'w').write( + 'q072210025 0 doc072207504499 1 16.214817060525405 pl2-baseline\nq072210025 0 doc072212607743 2 14.878122569655583 pl2-baseline') + + Path('tira-root/data/runs/dataset-1/example_participant/run-5-example_participant-eval/output').mkdir(parents=True, + exist_ok=True) + gzip.open('tira-root/data/runs/dataset-1/example_participant/run-5-example_participant-eval/output/.data-top-10-for-rendering.jsonl.gz', 'wt').write( + '{"queries": {"q072210025": {"qid": "q072210025","query": "recipe spring roll","original_query": {"query_id": "q072210025","text": "recipe spring roll"}}}, "documents": {}, "qrels": {}}') + d = modeldb.Dataset.objects.get(dataset_id=dataset_2) for i in range(2): for participant in ['participant-1', 'participant-2']: From 18079043f1a13183bee01172694af26818259b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Tue, 21 Nov 2023 22:10:57 +0100 Subject: [PATCH 15/18] Create new dev-docker image --- application/Dockerfile.application | 14 +++----- application/Dockerfile.application-dev | 36 ++++++++++++------- application/Makefile | 4 +-- application/requirements.txt | 3 +- application/src/tira/endpoints/diffir_api.py | 4 ++- application/src/tira/endpoints/serp_api.py | 3 +- .../test/diffir_tests/test_diffir_endpoint.py | 4 +-- application/test/run_all_tests.sh | 8 ++--- 8 files changed, 43 insertions(+), 33 deletions(-) diff --git a/application/Dockerfile.application b/application/Dockerfile.application index eda00496d..1e3d51f32 100644 --- a/application/Dockerfile.application +++ b/application/Dockerfile.application @@ -9,7 +9,7 @@ RUN cd /src/tira/frontend-vuetify \ # Only change in case of new / updated dependencies -FROM webis/tira-application:basis-0.0.76 +FROM webis/tira-application:basis-0.0.93 # This Dockerfile ensures that all dependencies do rarely change by starting from a basis image # that contains already all dependencies (so that the minor versions do rarely change, but we @@ -19,24 +19,20 @@ COPY application /tira/application WORKDIR /tira/application/src -RUN cd /tira/application/src/tira/frontend/ && \ - ln -s /tira-dependencies/node_modules/ node_modules && \ - npm run build && \ - mkdir -p /tira/model/src && \ - cd /tira/application/src && \ +RUN cd /tira/application/src && \ chown tira:tira -R /tira/application && \ python3 manage.py collectstatic && \ chmod +x /tira/application/src/tira/endpoints/aha && \ cp /tira/application/src/tira/management/commands/irds_cli.sh /irds_cli.sh && \ rm -f ./config/settings.yml ./config/config.yml ./config/tira-application-config.dev.yml && \ - rm -Rf tira/static/tira/frontend-vuetify/ && rm -Rf /tira/application/src/tira/static/tira/frontend-vuetify/ && \ - apk add sudo && \ - echo '%podman ALL=(ALL) ALL' > /etc/sudoers.d/podman + rm -Rf tira/static/tira/frontend-vuetify/ && rm -Rf /tira/application/src/tira/static/tira/frontend-vuetify/ COPY --from=0 /src/tira/static/tira/frontend-vuetify/ /tira/application/src/tira/static/tira/frontend-vuetify/ RUN cd /tira/application/ && \ ./test/run_all_tests.sh +USER tira + CMD sh -c "nginx && uwsgi --uid 1010 --gid 1010 --ini /tira/application/src/uwsgi.ini" diff --git a/application/Dockerfile.application-dev b/application/Dockerfile.application-dev index 5a96ba22a..63d6f8bf6 100644 --- a/application/Dockerfile.application-dev +++ b/application/Dockerfile.application-dev @@ -1,14 +1,24 @@ -# docker build -t webis/tira-application:basis-0.0.76 -f f.Dockerfile . -FROM webis/tira-application:basis-0.0.75 - -ENV PATH="${PATH}:/root/.cargo/bin" - -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > foo.sh && \ - sh ./foo.sh -y && \ - rm foo.sh && \ - pip3 install coverage coverage-badge && \ - apk add libffi-dev && \ - pip3 uninstall -y discourse-client-in-disraptor && \ - pip3 uninstall -y PyGithub && \ - pip3 install discourse-client-in-disraptor PyGithub==1.59.1 +# docker build -t webis/tira-application:basis-0.0.93 -f Dockerfile.application-dev . +FROM ubuntu:22.04 + +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y python3 python3-pip python-is-python3 mysql-client nginx sudo podman git \ + && apt-get install -y gcc default-libmysqlclient-dev pkg-config \ + && rm -Rf /var/cache/apt \ + && rm -Rf /root/.cache/pip \ + && echo '%podman ALL=(ALL) ALL' > /etc/sudoers.d/podman + +ADD requirements.txt /requirements.txt + +RUN pip3 install -r /requirements.txt \ + && rm -Rf /var/cache/apt \ + && rm -Rf /root/.cache/pip + +RUN useradd -u 1010 tira \ + && mkdir -p /tira/model/src \ + && chown tira:tira -R /tira + +RUN pip3 uninstall -y diffir \ + && pip3 install git+https://github.com/mam10eks/diffir diff --git a/application/Makefile b/application/Makefile index 9ae9c9735..70dcd9032 100644 --- a/application/Makefile +++ b/application/Makefile @@ -2,8 +2,8 @@ VENV_NAME?=venv PYTHON=${PWD}/${VENV_NAME}/bin/python3 -VERSION_APPLICATION=0.0.92 -VERSION_GRPC=0.0.92 +VERSION_APPLICATION=0.0.93 +VERSION_GRPC=0.0.93 .DEFAULT: help help: diff --git a/application/requirements.txt b/application/requirements.txt index 0f0956b91..a58375150 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -14,9 +14,10 @@ python-gitlab GitPython python-slugify git+https://github.com/allenai/ir_datasets +git+https://github.com/mam10eks/diffir pandas markdown -PyGithub +PyGithub==1.59.1 ghapi parameterized mockito diff --git a/application/src/tira/endpoints/diffir_api.py b/application/src/tira/endpoints/diffir_api.py index 1bfb63a24..bb54000a6 100644 --- a/application/src/tira/endpoints/diffir_api.py +++ b/application/src/tira/endpoints/diffir_api.py @@ -14,12 +14,14 @@ def doc_file_for_run(vm_id, dataset_id, task_id, run_id): + checked_paths = [] for evaluation in model.get_evaluations_of_run(vm_id, run_id): for f in [".data-top-10-for-rendering.jsonl.gz", ".data-top-10-for-rendering.jsonl"]: p = Path(settings.TIRA_ROOT) / "data" / "runs" / dataset_id / vm_id / evaluation / 'output' / f + checked_paths += [str(p)] if p.is_file(): return p - raise ValueError('.data-top-10-for-rendering.jsonl') + raise ValueError(f'Could not find .data-top-10-for-rendering.jsonl. Searched in {checked_paths}.') def load_irds_metadata_of_task(task, dataset): diff --git a/application/src/tira/endpoints/serp_api.py b/application/src/tira/endpoints/serp_api.py index 5eea99e54..300c8c5ac 100644 --- a/application/src/tira/endpoints/serp_api.py +++ b/application/src/tira/endpoints/serp_api.py @@ -29,7 +29,8 @@ def serp(request, context, vm_id, dataset_id, task_id, run_id, topk): try: from diffir.run import diff_from_local_data - except: + except Exception as e: + logger.exception(e) raise ValueError('Could not load dependency diffir') doc_file = doc_file_for_run(vm_id, dataset_id, task_id, run_id) diff --git a/application/test/diffir_tests/test_diffir_endpoint.py b/application/test/diffir_tests/test_diffir_endpoint.py index c369e56d7..1103a2aff 100644 --- a/application/test/diffir_tests/test_diffir_endpoint.py +++ b/application/test/diffir_tests/test_diffir_endpoint.py @@ -19,7 +19,7 @@ def test_diffir_with_json(self): actual = diffir(request, vm_id='example_participant', dataset_id='dataset-1', task_id='t1', run_id='run-3-example_participant', topk=10) # Assert - self.assertTrue('dsa', actual.content.decode('utf-8').startswith('')) + self.assertTrue(actual.content.decode('utf-8').startswith('')) def test_diffir_with_json_gz(self): # Arrange @@ -29,7 +29,7 @@ def test_diffir_with_json_gz(self): actual = diffir(request, vm_id='example_participant', dataset_id='dataset-1', task_id='t1', run_id='run-5-example_participant', topk=10) # Assert - self.assertTrue('dsa', actual.content.decode('utf-8').startswith('')) + self.assertTrue(actual.content.decode('utf-8').startswith('')) @classmethod def tearDownClass(cls): diff --git a/application/test/run_all_tests.sh b/application/test/run_all_tests.sh index 446a7854a..5b98bd91b 100755 --- a/application/test/run_all_tests.sh +++ b/application/test/run_all_tests.sh @@ -13,9 +13,9 @@ python3 ../src/manage.py makemigrations tira python3 ../src/manage.py migrate tira coverage run --data-file=test-coverage/.coverage ../src/manage.py test --settings=settings_test --failfast -coverage report --data-file=test-coverage/.coverage > test-coverage/coverage-report -cd test-coverage +#coverage report --data-file=test-coverage/.coverage > test-coverage/coverage-report +#cd test-coverage -rm -Rf coverage.svg -coverage-badge -o coverage.svg +#rm -Rf coverage.svg +#coverage-badge -o coverage.svg From e0785c2a6bf8076d39168873b941347d729ab9bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Tue, 21 Nov 2023 23:13:11 +0100 Subject: [PATCH 16/18] Create new dev-docker image --- application/Dockerfile.application | 2 - application/Dockerfile.application-dev | 39 +++++++++++++++++--- application/src/tira/endpoints/diffir_api.py | 2 +- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/application/Dockerfile.application b/application/Dockerfile.application index 1e3d51f32..3079eae6a 100644 --- a/application/Dockerfile.application +++ b/application/Dockerfile.application @@ -32,7 +32,5 @@ COPY --from=0 /src/tira/static/tira/frontend-vuetify/ /tira/application/src/tira RUN cd /tira/application/ && \ ./test/run_all_tests.sh -USER tira - CMD sh -c "nginx && uwsgi --uid 1010 --gid 1010 --ini /tira/application/src/uwsgi.ini" diff --git a/application/Dockerfile.application-dev b/application/Dockerfile.application-dev index 63d6f8bf6..5a3718a54 100644 --- a/application/Dockerfile.application-dev +++ b/application/Dockerfile.application-dev @@ -15,10 +15,39 @@ RUN pip3 install -r /requirements.txt \ && rm -Rf /var/cache/apt \ && rm -Rf /root/.cache/pip -RUN useradd -u 1010 tira \ - && mkdir -p /tira/model/src \ - && chown tira:tira -R /tira - RUN pip3 uninstall -y diffir \ - && pip3 install git+https://github.com/mam10eks/diffir + && pip3 install git+https://github.com/mam10eks/diffir && \ + addgroup --gid 1010 tira && \ + adduser --disabled-password --uid 1010 --ingroup tira tira && \ + adduser tira root && \ + mkdir /tira-tmp-debug && \ + mkdir -p /tira/application/src/tira && \ + mkdir -p /etc/nginx/logs && \ + mkdir -p /mnt/ceph/tira && \ + mkdir -p /root/.ir_datasets \ + && rm -Rf /var/cache/apt \ + && rm -Rf /root/.cache/pip \ + && apt-get install -y fuse-overlayfs + +COPY src/tira_nginx.conf /etc/nginx/nginx.conf + +RUN chown -R tira:tira /usr/local && \ + chmod 777 /var/run/ && \ + echo "tira ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers &&\ + touch /var/log/tira_debug.txt &&\ + touch /tira/application/src/debug.log &&\ + chown tira:tira /var/log/tira_debug.txt &&\ + chown tira:tira /tira/application/src/debug.log &&\ + chown -R tira:tira /root/.ir_datasets &&\ + chmod 777 /root/.ir_datasets &&\ + chmod 777 /root &&\ + mkdir -p /home/tira/.tira &&\ + mkdir -p /mnt/ceph/tira/log/tira-application &&\ + mkdir /home/tira/.ssh &&\ + mkdir /mnt/ceph/tira/state &&\ + mkdir -p /usr/share/nginx/logs/ && \ + chmod 777 /usr/share/nginx/logs/ && \ + touch /mnt/ceph/tira/state/tira_vm_states.sqlite3 &&\ + chown tira:tira -R /home/tira &&\ + chown tira:tira -R /tira diff --git a/application/src/tira/endpoints/diffir_api.py b/application/src/tira/endpoints/diffir_api.py index bb54000a6..06e726b7f 100644 --- a/application/src/tira/endpoints/diffir_api.py +++ b/application/src/tira/endpoints/diffir_api.py @@ -16,7 +16,7 @@ def doc_file_for_run(vm_id, dataset_id, task_id, run_id): checked_paths = [] for evaluation in model.get_evaluations_of_run(vm_id, run_id): - for f in [".data-top-10-for-rendering.jsonl.gz", ".data-top-10-for-rendering.jsonl"]: + for f in [".data-top-10-for-rendering.jsonl.gz", ".data-top-10-for-rendering.jsonl", ".data-top-10-for-rendering.json.gz", ".data-top-10-for-rendering.json"]: p = Path(settings.TIRA_ROOT) / "data" / "runs" / dataset_id / vm_id / evaluation / 'output' / f checked_paths += [str(p)] if p.is_file(): From f624a8ea265cf8fa919f6b5a0f72efd02fce0800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Wed, 22 Nov 2023 05:29:53 +0100 Subject: [PATCH 17/18] add dev containers --- application/.devcontainer.json | 8 ++++++++ application/Makefile | 13 ++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 application/.devcontainer.json diff --git a/application/.devcontainer.json b/application/.devcontainer.json new file mode 100644 index 000000000..b4700a90b --- /dev/null +++ b/application/.devcontainer.json @@ -0,0 +1,8 @@ +{ + "image": "webis/tira-application:basis-0.0.93", + "customizations": { + "vscode": { + "extensions": ["ms-python.python", "ms-python.vscode-pylance", "ms-toolsai.jupyter"] + } + } +} diff --git a/application/Makefile b/application/Makefile index 70dcd9032..7d49add8a 100644 --- a/application/Makefile +++ b/application/Makefile @@ -70,9 +70,8 @@ src/tira/frontend/webpack-stats.json: npm install; \ npm run build -tests: venv src/tira/frontend/webpack-stats.json - @. ./venv/bin/activate \ - && ./test/run_all_tests.sh +tests: + ./test/run_all_tests.sh # Requirements are in setup.py, so whenever setup.py is changed, re-run installation of dependencies. @@ -82,10 +81,10 @@ venv: python3 -m pip install wheel && \ python3 -m pip install -r requirements.txt" -run-develop: venv - $(VENV_NAME)/bin/python src/manage.py makemigrations \ - && $(VENV_NAME)/bin/python src/manage.py migrate --fake \ - && $(VENV_NAME)/bin/python src/manage.py run_develop +run-develop: + python src/manage.py makemigrations \ + && python src/manage.py migrate --fake \ + && python src/manage.py run_develop vite-build: docker run -v ${PWD}:/app --platform linux/amd64 --rm -ti -w /app/src/tira/frontend-vuetify --entrypoint yarn webis/tira:vuetify-dev-0.0.1 build From eb1a27830ffa241c23ad506d60ebfa05f0499124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maik=20Fr=C3=B6be?= Date: Wed, 22 Nov 2023 08:30:52 +0100 Subject: [PATCH 18/18] Add dev container --- README.md | 20 ++++------------- application/src/tira/data/HybridDatabase.py | 25 ++++++++++++++++++--- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 31f69e518..535654164 100644 --- a/README.md +++ b/README.md @@ -12,20 +12,14 @@ Components: ## Setup Your Local Development Environment +We use [dev containers](https://code.visualstudio.com/docs/devcontainers/containers) to simplify development. Please install Docker and an IDE with support for dev containers on your machine (we usually use VS Code). + First, please clone the repository: ``` git clone git@github.com:tira-io/tira.git ``` -Please change your directory to `application`: -``` -cd application -``` - -Install your virtual environment via: -``` -make setup -``` +Please open the directory `application` in VS Code, and confirm to use the provided dev container. If you want to work on production data, please ensure that you can login to ssh.webis.de, and then do the following: @@ -33,13 +27,7 @@ If you want to work on production data, please ensure that you can login to ssh. make import-data-from-dump ``` -Then, to start TIRA locally, please start: - -``` -make webpack-watch -``` - -and +To start TIRA locally, please run: ``` make run-develop diff --git a/application/src/tira/data/HybridDatabase.py b/application/src/tira/data/HybridDatabase.py index f77707655..046fa9a4b 100644 --- a/application/src/tira/data/HybridDatabase.py +++ b/application/src/tira/data/HybridDatabase.py @@ -8,6 +8,7 @@ import randomname import os import zipfile +import json from tira.util import TiraModelWriteError, TiraModelIntegrityError from tira.proto import TiraClientWebMessages_pb2 as modelpb @@ -993,7 +994,8 @@ def get_evaluations_with_keys_by_dataset(self, dataset_id, include_unpublished=F tira_evaluation_review.blinded, tira_run_review.published, tira_run_review.blinded, tira_evaluation.measure_key, tira_evaluation.measure_value, tira_run_review.reviewer_id, tira_run_review.no_errors, tira_run_review.has_errors, tira_run_review.has_no_errors, - tira_evaluation_review.reviewer_id, tira_run_review.reviewer_id + tira_evaluation_review.reviewer_id, tira_run_review.reviewer_id, + tira_linktosoftwaresubmissiongitrepository.build_environment FROM tira_run as evaluation_run INNER JOIN @@ -1004,6 +1006,8 @@ def get_evaluations_with_keys_by_dataset(self, dataset_id, include_unpublished=F tira_software ON input_run.software_id = tira_software.id LEFT JOIN tira_dockersoftware ON input_run.docker_software_id = tira_dockersoftware.docker_software_id + LEFT JOIN + tira_linktosoftwaresubmissiongitrepository ON tira_dockersoftware.docker_software_id = tira_linktosoftwaresubmissiongitrepository.docker_software_id LEFT JOIN tira_review as tira_evaluation_review ON evaluation_run.run_id = tira_evaluation_review.run_id LEFT JOIN @@ -1033,7 +1037,21 @@ def get_evaluations_with_keys_by_dataset(self, dataset_id, include_unpublished=F return self.__parse_submissions(rows, include_unpublished, round_floats, show_only_unreviewed, show_only_unreviewed) @staticmethod - def __parse_submissions(rows, include_unpublished, round_floats, include_without_evaluation=False, show_only_unreviewed=False): + def __link_to_code(build_environment): + if not build_environment: + return None + + build_environment = json.loads(build_environment) + + if 'TIRA_JUPYTER_NOTEBOOK' not in build_environment or 'GITHUB_REPOSITORY' not in build_environment or 'GITHUB_WORKFLOW' not in build_environment or 'GITHUB_SHA' not in build_environment: + return None + + if build_environment['GITHUB_WORKFLOW'] == ".github/workflows/upload-notebook-submission.yml": + return f'https://github.com/{build_environment["GITHUB_REPOSITORY"]}/tree/{build_environment["GITHUB_SHA"]}/jupyter-notebook-submissions/{build_environment["TIRA_JUPYTER_NOTEBOOK"]}' + + return None + + def __parse_submissions(self, rows, include_unpublished, round_floats, include_without_evaluation=False, show_only_unreviewed=False): keys = dict() input_run_to_evaluation = {} @@ -1047,7 +1065,7 @@ def round_if_float(fl): for dataset_id, run_id, input_run_id, upload_display_name, upload_vm_id, software_vm_id, docker_display_name, \ docker_vm_id, eval_published, eval_blinded, run_published, run_blinded, m_key, m_value, \ - reviewer_id, no_errors, has_errors, has_no_errors, tira_evaluation_reviewer_id, tira_run_reviewer_id in rows: + reviewer_id, no_errors, has_errors, has_no_errors, tira_evaluation_reviewer_id, tira_run_reviewer_id, build_environment in rows: if (not include_without_evaluation and not m_key) or (not include_unpublished and not eval_published): continue @@ -1088,6 +1106,7 @@ def round_if_float(fl): input_run_to_evaluation[run_id]['is_upload'] = is_upload input_run_to_evaluation[run_id]['is_software'] = is_software input_run_to_evaluation[run_id]['review_state'] = review_state + input_run_to_evaluation[run_id]['link_to_code'] = self.__link_to_code(build_environment) if m_key: input_run_to_evaluation[run_id]['measures'][m_key] = m_value