From 4359ddd4ffba6c47aa652a3f81ffa066fe2b709e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 10:29:26 +0100 Subject: [PATCH 01/24] Some minor changes as some stuff is deprecated in Python 3.10 (or will be in the future and causes deprecation warnings) --- loadskernel/io_functions/read_bdf.py | 2 +- loadskernel/io_functions/read_mona.py | 6 +++--- loadskernel/plotting_standard.py | 4 ++-- loadskernel/spline_rules.py | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/loadskernel/io_functions/read_bdf.py b/loadskernel/io_functions/read_bdf.py index 1c087ebe..2f5d7d97 100644 --- a/loadskernel/io_functions/read_bdf.py +++ b/loadskernel/io_functions/read_bdf.py @@ -190,7 +190,7 @@ def aggregate_cards(self, card_names): for card_name in card_names: old_size = self.cards[card_name].shape[0] sort_by_field = self.card_interpreters[card_name].field_names[0] - self.cards[card_name] = self.cards[card_name].groupby(by=sort_by_field, as_index=False).agg(sum) + self.cards[card_name] = self.cards[card_name].groupby(by=sort_by_field, as_index=False).agg("sum") new_size = self.cards[card_name].shape[0] if old_size != new_size: logging.info('Aggregating {} {}s'.format(old_size - new_size, card_name)) diff --git a/loadskernel/io_functions/read_mona.py b/loadskernel/io_functions/read_mona.py index ef1a290c..75c6cd8f 100644 --- a/loadskernel/io_functions/read_mona.py +++ b/loadskernel/io_functions/read_mona.py @@ -416,7 +416,7 @@ def add_SET1(pandas_sets): set_values = [] for _, row in pandas_sets[['values']].iterrows(): # create a copy of the current row to work with - my_row = copy.deepcopy(row[0]) + my_row = copy.deepcopy(row.iloc[0]) # remove all None values my_row = [item for item in my_row if item is not None] values = [] @@ -448,8 +448,8 @@ def add_AECOMP(pandas_aecomps): # Loop over the rows to check for NaNs and None, which occur in case an empty field was in the list. # Then, select only the valid list items. for _, row in pandas_aecomps[['LISTID']].iterrows(): - is_id = [pd.notna(x) for x in row[0]] - list_id.append(list(compress(row[0], is_id))) + is_id = [pd.notna(x) for x in row.iloc[0]] + list_id.append(list(compress(row.iloc[0], is_id))) aecomp = {} aecomp['name'] = pandas_aecomps['NAME'].to_list() diff --git a/loadskernel/plotting_standard.py b/loadskernel/plotting_standard.py index efb1b271..d6c32e7c 100755 --- a/loadskernel/plotting_standard.py +++ b/loadskernel/plotting_standard.py @@ -166,7 +166,7 @@ def plot_forces_deformation_interactive(self): def plot_monstations(self, filename_pdf): # launch plotting - self.pp = PdfPages(filename_pdf) + self.pp = PdfPages(filename_pdf, keep_empty=False) self.potato_plots() if self.cuttingforces_wing: self.cuttingforces_along_wing_plots() @@ -570,7 +570,7 @@ def plot_eigenvalues(self): ax[1].minorticks_on() ax[1].axis([-1.0, 1.0, imin, imax]) # connect with y-axis from left hand plot - ax[0].get_shared_y_axes().join(ax[0], ax[1]) + ax[0].sharey(ax[1]) ax[1].yaxis.set_tick_params(which='both', labelleft=False, labelright=False) ax[1].yaxis.offsetText.set_visible(False) # add legend diff --git a/loadskernel/spline_rules.py b/loadskernel/spline_rules.py index 91cf4857..bde8a1b2 100644 --- a/loadskernel/spline_rules.py +++ b/loadskernel/spline_rules.py @@ -32,7 +32,8 @@ def rules_point(grid_i, grid_d): # All dependent grids are mapped to one grid point, which might be CG or MAC # Assumption: the relevant point is expected to be the only/first point in the independet grid splinerules = {} - splinerules[int(grid_i['ID'])] = list(grid_d['ID']) + assert len(grid_i['ID']) == 1, "The independent grid 'grid_i' may have only one grid point for this kind of spline rules." + splinerules[grid_i['ID'][0]] = list(grid_d['ID']) return splinerules From 6310295061998c5ebbeeedc5115c9bb3a39a2857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 10:32:39 +0100 Subject: [PATCH 02/24] Add a filter to pytest to avoid annoying deprecation warnings raised by third party code --- .pytest.ini | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .pytest.ini diff --git a/.pytest.ini b/.pytest.ini new file mode 100644 index 00000000..c3d18474 --- /dev/null +++ b/.pytest.ini @@ -0,0 +1,10 @@ +[pytest] +filterwarnings = + # Filter to avoid annoying deprecation warnings raised by third party code + # Step 1: ignore all deprecation warnings + ignore::DeprecationWarning:: + # Step 2: re-activate deprecation warnings for own modules + default::DeprecationWarning:loadskernel.* + default::DeprecationWarning:loadsviewer.* + default::DeprecationWarning:modelviewer.* + default::DeprecationWarning:tests.* \ No newline at end of file From 9825de4de98f5f4b831b61af344f2ea0f44fe406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 10:41:55 +0100 Subject: [PATCH 03/24] Adapt GitLab and GitHub workflows to Python 3.10 --- .github/workflows/coding-style.yml | 4 ++-- .github/workflows/regression-tests.yml | 6 +++--- .gitlab-ci.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coding-style.yml b/.github/workflows/coding-style.yml index ac53f481..7e0c404c 100644 --- a/.github/workflows/coding-style.yml +++ b/.github/workflows/coding-style.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.8"] + python-version: ["3.8", "3.10"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -41,7 +41,7 @@ jobs: strategy: matrix: # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.8"] + python-version: ["3.8", "3.10"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 436a2803..9056e050 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.8"] + python-version: ["3.8", "3.10"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -38,8 +38,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.8"] + # Select Python version to be used for compiling here. + python-version: ["3.10"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d3f57367..9744cc2b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ stages: - deploy .virtenv: &virtualenv - - source /work/voss_ar/Software/miniconda3/etc/profile.d/conda.sh + - source /work/voss_ar/Software/miniconda-3.10/etc/profile.d/conda.sh - conda activate # To make things faster, re-use existing site packages. - python -m venv virtualenv --system-site-packages From c75249e1361abdab0a2ab4e49c3c00f105509802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 18:17:53 +0100 Subject: [PATCH 04/24] GitLab and GitHub CI: add check if module can be imported after installation, remove old MPI stuff --- .github/workflows/regression-tests.yml | 1 + .gitlab-ci.yml | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 436a2803..cb87ac15 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -30,6 +30,7 @@ jobs: # Install with -e (in editable mode) to allow the tracking of the test coverage pip install -e . # Check result of installation + python -c "import loadskernel" which loads-kernel which model-viewer which loads-compare diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d3f57367..d985eee5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,9 +17,7 @@ stages: # Check python version - which python - which pytest - # Set-up MPI - - export PATH=/work/voss_ar/Software/mpich-3.4.2/bin:$PATH - - export LD_LIBRARY_PATH=/work/voss_ar/Software/mpich-3.4.2/lib + # Check MPI - which mpiexec # Check location - pwd @@ -34,6 +32,7 @@ pip-installation: - *virtualenv - pip install -e . # Check result of installation + - python -c "import loadskernel" - which loads-kernel - which model-viewer - which loads-compare From 1881c3821a394cce1ac5fafb89f1ca0b37783ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 18:51:48 +0100 Subject: [PATCH 05/24] Better to upgrade directly to Python 3.11 --- .github/workflows/coding-style.yml | 4 ++-- .github/workflows/regression-tests.yml | 4 ++-- .gitlab-ci.yml | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coding-style.yml b/.github/workflows/coding-style.yml index 7e0c404c..e69adb6d 100644 --- a/.github/workflows/coding-style.yml +++ b/.github/workflows/coding-style.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.8", "3.10"] + python-version: ["3.8", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -41,7 +41,7 @@ jobs: strategy: matrix: # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.8", "3.10"] + python-version: ["3.8", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 9056e050..37970a20 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.8", "3.10"] + python-version: ["3.8", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -39,7 +39,7 @@ jobs: strategy: matrix: # Select Python version to be used for compiling here. - python-version: ["3.10"] + python-version: ["3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9744cc2b..a37617af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ stages: - deploy .virtenv: &virtualenv - - source /work/voss_ar/Software/miniconda-3.10/etc/profile.d/conda.sh + - source /work/f_jwsb/software/miniconda-3.11/etc/profile.d/conda.sh - conda activate # To make things faster, re-use existing site packages. - python -m venv virtualenv --system-site-packages @@ -86,10 +86,11 @@ Pytest: - *virtualenv # Install with -e (in editable mode) to allow the tracking of the test coverage - pip install -e . + - pip list # Get the examples repository - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.dlr.de/loads-kernel/loads-kernel-examples.git # Run the actual testing of the code with pytest - - pytest -v --basetemp=./test_tmp --cov=loadskernel --cov=modelviewer --cov=loadscompare --junitxml=testresult.xml + - pytest -v --basetemp=./tmp --cov=loadskernel --cov=modelviewer --cov=loadscompare --junitxml=testresult.xml # Create some reports - coverage report - coverage xml -o coverage.xml From 11c9c1758c7cfe40e17f76ba039e712b3a988885 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 19:11:55 +0100 Subject: [PATCH 06/24] Using python -m pytest is necessary because pytest has the habit of not looking in the site-packages of the venv --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 764eba57..c336ecce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -89,7 +89,8 @@ Pytest: # Get the examples repository - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.dlr.de/loads-kernel/loads-kernel-examples.git # Run the actual testing of the code with pytest - - pytest -v --basetemp=./tmp --cov=loadskernel --cov=modelviewer --cov=loadscompare --junitxml=testresult.xml + # Using python -m pytest is necessary because pytest has the habit of not looking in the site-packages of the venv + - python -m pytest -v --basetemp=./tmp --cov=loadskernel --cov=modelviewer --cov=loadscompare --junitxml=testresult.xml # Create some reports - coverage report - coverage xml -o coverage.xml From 59ab771a23559989e9bb769851400fb90b5ee58a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 19:59:12 +0100 Subject: [PATCH 07/24] Use conda provided by $CONDA --- .github/workflows/coding-style.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coding-style.yml b/.github/workflows/coding-style.yml index e69adb6d..d2ae152e 100644 --- a/.github/workflows/coding-style.yml +++ b/.github/workflows/coding-style.yml @@ -50,8 +50,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install pylint + source $CONDA/etc/profile.d/conda.sh + conda activate + conda install -y -c conda-forge pylint + #python -m pip install --upgrade pip + #pip install pylint # Install the package itself to make sure that all imports work. pip install . - name: Analysing the code with pylint From 27db7eedf393ef96a7d09174ede1fdb4a141f9e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 20:01:58 +0100 Subject: [PATCH 08/24] Install everything with conda --- .github/workflows/coding-style.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coding-style.yml b/.github/workflows/coding-style.yml index d2ae152e..6d655b55 100644 --- a/.github/workflows/coding-style.yml +++ b/.github/workflows/coding-style.yml @@ -52,7 +52,7 @@ jobs: run: | source $CONDA/etc/profile.d/conda.sh conda activate - conda install -y -c conda-forge pylint + conda install -y -c conda-forge mayavi traits traitsui pyface pyfmi h5py mpi4py pytest-cov psutil pytables pyyaml matplotlib mamba pandas openpyxl jupyter jupyter-book flake8 pylint #python -m pip install --upgrade pip #pip install pylint # Install the package itself to make sure that all imports work. From 2c2cab9f7f65abee8c6e81768c5ea39f87cff4e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 20:05:08 +0100 Subject: [PATCH 09/24] Source conda again --- .github/workflows/coding-style.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/coding-style.yml b/.github/workflows/coding-style.yml index 6d655b55..961a11ee 100644 --- a/.github/workflows/coding-style.yml +++ b/.github/workflows/coding-style.yml @@ -59,4 +59,6 @@ jobs: pip install . - name: Analysing the code with pylint run: | + source $CONDA/etc/profile.d/conda.sh + conda activate pylint $(git ls-files '*.py') --fail-under=7.0 \ No newline at end of file From 1b9b2f347aab8aa2f3aa1c80cfcf101fb8394bd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 20:33:58 +0100 Subject: [PATCH 10/24] Use requiremets.txt for tests --- .github/workflows/coding-style.yml | 8 +++----- tests/requirements.txt | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 tests/requirements.txt diff --git a/.github/workflows/coding-style.yml b/.github/workflows/coding-style.yml index 961a11ee..6d28a5a8 100644 --- a/.github/workflows/coding-style.yml +++ b/.github/workflows/coding-style.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.8", "3.11"] + python-version: ["3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -41,7 +41,7 @@ jobs: strategy: matrix: # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.8", "3.11"] + python-version: ["3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -52,9 +52,7 @@ jobs: run: | source $CONDA/etc/profile.d/conda.sh conda activate - conda install -y -c conda-forge mayavi traits traitsui pyface pyfmi h5py mpi4py pytest-cov psutil pytables pyyaml matplotlib mamba pandas openpyxl jupyter jupyter-book flake8 pylint - #python -m pip install --upgrade pip - #pip install pylint + conda install -y -c conda-forge --file ./tests/requirements.txt # Install the package itself to make sure that all imports work. pip install . - name: Analysing the code with pylint diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000..1ed6a6c5 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,20 @@ +# These are the requirements to be used during regression testing +mayavi +traits +traitsui +pyface +pyfmi +h5py +mpi4py +pytest-cov +psutil +pytables +pyyaml +matplotlib +mamba +pandas +openpyxl +jupyter +jupyter-book +flake8 +pylint \ No newline at end of file From 4de9cf30b0e30888e15ce0377e0e173e6d18e65d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 20:49:57 +0100 Subject: [PATCH 11/24] Aplly conda installation to regression tests --- .github/workflows/coding-style.yml | 5 +++-- .github/workflows/regression-tests.yml | 12 ++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coding-style.yml b/.github/workflows/coding-style.yml index 6d28a5a8..9f28bbf6 100644 --- a/.github/workflows/coding-style.yml +++ b/.github/workflows/coding-style.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Add multiple Python versions here to run tests on new(er) versions. + # Add the Python versions here to run tests on new(er) versions. python-version: ["3.11"] steps: - uses: actions/checkout@v3 @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Add multiple Python versions here to run tests on new(er) versions. + # Add multiple Python versions here to run tests on new(er) versions. python-version: ["3.11"] steps: - uses: actions/checkout@v3 @@ -50,6 +50,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | + # Install same requirements as to be used during regression testing source $CONDA/etc/profile.d/conda.sh conda activate conda install -y -c conda-forge --file ./tests/requirements.txt diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 3d66c298..14088e79 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -10,14 +10,15 @@ on: branches: '*' jobs: - pip-installation: + pure-pip-installation: # This stage only tests if the installation is possible. # The evironment created herein will be discared and re-created in the test stage. runs-on: ubuntu-latest strategy: matrix: # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.8", "3.11"] + python-version: ["3.8", "3.10"] + fail-fast: false steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -49,8 +50,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install jupyter-book + # Install same requirements as to be used during regression testing + source $CONDA/etc/profile.d/conda.sh + conda activate + conda install -y -c conda-forge --file ./tests/requirements.txt + # Install the package itself to make sure that all imports work. pip install . - name: Assemble the tutorials to a jupyter book and build htlm pages run: | From 7abc2904bfb7af2acf9122668e353455799c6b37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 21:01:28 +0100 Subject: [PATCH 12/24] Fix Jupyter, add Pytest --- .github/workflows/regression-tests.yml | 50 ++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 14088e79..c2fe01a5 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -36,6 +36,54 @@ jobs: which model-viewer which loads-compare + Pytest: + runs-on: ubuntu-latest + strategy: + matrix: + # Add multiple Python versions here to run tests on new(er) versions. + python-version: ["3.11"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + # Install same requirements as to be used during regression testing + source $CONDA/etc/profile.d/conda.sh + conda activate + conda install -y -c conda-forge --file ./tests/requirements.txt + # Install the package itself to make sure that all imports work. + pip install -e . + - name: Analysing the code with pytest + run: | + source $CONDA/etc/profile.d/conda.sh + conda activate + # Run the actual testing of the code with pytest + # Using python -m pytest is necessary because pytest has the habit of not looking in the site-packages of the venv + python -m pytest -v --basetemp=./tmp -k test_gui --cov=loadskernel --cov=modelviewer --cov=loadscompare --junitxml=testresult.xml + # Create some reports + coverage report + coverage xml -o coverage.xml + coverage html --directory ./coverage/coverage + - name: Upload test restults and coverage as an artifact + uses: actions/upload-artifact@v3 + with: + name: test results and coverage + path: | + testresult.xml + coverage.xml + coverage + if-no-files-found: ignore + - name: Upload Jupyter book for pages + # This is not a normal artifact but one that can be deployed to the GitHub pages in the next step + uses: actions/upload-pages-artifact@v3 + with: + name: github-pages # This name may not be changed according to the documentation + path: ./coverage + if-no-files-found: ignore + Jupyter: runs-on: ubuntu-latest strategy: @@ -58,6 +106,8 @@ jobs: pip install . - name: Assemble the tutorials to a jupyter book and build htlm pages run: | + source $CONDA/etc/profile.d/conda.sh + conda activate jupyter-book build ./doc/tutorials # Put the html into a 2nd-level sub-folder and use 1st-level subfolder for uploading mkdir ./doc/html From d68a98591192b7baaf3f394c30da3ae44537d6ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Wed, 10 Jan 2024 21:07:21 +0100 Subject: [PATCH 13/24] Restore the dummy test --- .github/workflows/regression-tests.yml | 5 ++--- tests/test_dummy.py | 6 ++++++ 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 tests/test_dummy.py diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index c2fe01a5..c271562d 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -62,7 +62,7 @@ jobs: conda activate # Run the actual testing of the code with pytest # Using python -m pytest is necessary because pytest has the habit of not looking in the site-packages of the venv - python -m pytest -v --basetemp=./tmp -k test_gui --cov=loadskernel --cov=modelviewer --cov=loadscompare --junitxml=testresult.xml + python -m pytest -v --basetemp=./tmp -k test_dummy --cov=loadskernel --cov=modelviewer --cov=loadscompare --junitxml=testresult.xml # Create some reports coverage report coverage xml -o coverage.xml @@ -128,8 +128,7 @@ jobs: deploy-pages: # Add a dependency to the build job - needs: Jupyter - + needs: [Jupyter, Pytest] # Grant GITHUB_TOKEN the permissions required to make a Pages deployment permissions: pages: write # to deploy to Pages diff --git a/tests/test_dummy.py b/tests/test_dummy.py new file mode 100644 index 00000000..3313ff50 --- /dev/null +++ b/tests/test_dummy.py @@ -0,0 +1,6 @@ +import platform + +def test_dummy(): + print('The dummy test is executed.') + print('Running on python version {}'.format(platform.python_version())) + pass \ No newline at end of file From 75054bc7b5f54a9ad78aacd774240bd32bc146a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Thu, 11 Jan 2024 10:19:46 +0100 Subject: [PATCH 14/24] Fix test_gui and test_dummy --- .github/workflows/regression-tests.yml | 7 ------- tests/test_dummy.py | 3 ++- tests/test_gui.py | 7 +++++-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index c271562d..61966c52 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -76,13 +76,6 @@ jobs: coverage.xml coverage if-no-files-found: ignore - - name: Upload Jupyter book for pages - # This is not a normal artifact but one that can be deployed to the GitHub pages in the next step - uses: actions/upload-pages-artifact@v3 - with: - name: github-pages # This name may not be changed according to the documentation - path: ./coverage - if-no-files-found: ignore Jupyter: runs-on: ubuntu-latest diff --git a/tests/test_dummy.py b/tests/test_dummy.py index 3313ff50..19f01b4e 100644 --- a/tests/test_dummy.py +++ b/tests/test_dummy.py @@ -1,6 +1,7 @@ import platform + def test_dummy(): print('The dummy test is executed.') print('Running on python version {}'.format(platform.python_version())) - pass \ No newline at end of file + pass diff --git a/tests/test_gui.py b/tests/test_gui.py index 905a1d41..02c51eac 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -1,7 +1,10 @@ import logging -from loadscompare import compare -from modelviewer import view +try: + from loadscompare import compare + from modelviewer import view +except ImportError: + pass class TestLoadsCompare(): From 5034b07491d45b869b3c3366cffc4c3adb2ace82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Thu, 11 Jan 2024 10:28:44 +0100 Subject: [PATCH 15/24] add conda-and-pip-installation, revert to pip for pytest and jupyter --- .github/workflows/regression-tests.yml | 47 ++++++++++++++++++-------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 61966c52..9ae1e1cf 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.8", "3.10"] + python-version: ["3.10"] fail-fast: false steps: - uses: actions/checkout@v3 @@ -35,31 +35,55 @@ jobs: which loads-kernel which model-viewer which loads-compare - - Pytest: + + conda-and-pip-installation: + # This stage only tests if the installation is possible. + # The evironment created herein will be discared and re-created in the test stage. runs-on: ubuntu-latest strategy: matrix: - # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.11"] + # Add multiple Python versions here to run tests on new(er) versions. + python-version: ["3.10", "3.11"] + fail-fast: false steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Build and install run: | # Install same requirements as to be used during regression testing source $CONDA/etc/profile.d/conda.sh conda activate conda install -y -c conda-forge --file ./tests/requirements.txt + # Install with -e (in editable mode) to allow the tracking of the test coverage + pip install -e . + # Check result of installation + python -c "import loadskernel" + which loads-kernel + which model-viewer + which loads-compare + + Pytest: + runs-on: ubuntu-latest + strategy: + matrix: + # Add multiple Python versions here to run tests on new(er) versions. + python-version: ["3.10"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip # Install the package itself to make sure that all imports work. pip install -e . - name: Analysing the code with pytest run: | - source $CONDA/etc/profile.d/conda.sh - conda activate # Run the actual testing of the code with pytest # Using python -m pytest is necessary because pytest has the habit of not looking in the site-packages of the venv python -m pytest -v --basetemp=./tmp -k test_dummy --cov=loadskernel --cov=modelviewer --cov=loadscompare --junitxml=testresult.xml @@ -91,16 +115,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - # Install same requirements as to be used during regression testing - source $CONDA/etc/profile.d/conda.sh - conda activate - conda install -y -c conda-forge --file ./tests/requirements.txt + python -m pip install --upgrade pip # Install the package itself to make sure that all imports work. pip install . - name: Assemble the tutorials to a jupyter book and build htlm pages run: | - source $CONDA/etc/profile.d/conda.sh - conda activate jupyter-book build ./doc/tutorials # Put the html into a 2nd-level sub-folder and use 1st-level subfolder for uploading mkdir ./doc/html From d0307bc4db0d7ddf51ced7e54ef19736f5f28271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Thu, 11 Jan 2024 10:41:36 +0100 Subject: [PATCH 16/24] Run Jupyter with Python 3.10 --- .github/workflows/regression-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 9ae1e1cf..4e4464be 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -106,7 +106,7 @@ jobs: strategy: matrix: # Select Python version to be used for compiling here. - python-version: ["3.11"] + python-version: ["3.10"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From ad60890c5ff4340985d5174b55a9ab2ccffec77a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Thu, 11 Jan 2024 10:47:07 +0100 Subject: [PATCH 17/24] Add combine-pages --- .github/workflows/regression-tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 4e4464be..ae53b422 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -138,6 +138,16 @@ jobs: path: ./doc/html if-no-files-found: ignore + combine-pages: + # Add a dependency to the build job + needs: [Jupyter, Pytest] + steps: + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + - name: See what we've got + run: ls -la + deploy-pages: # Add a dependency to the build job needs: [Jupyter, Pytest] From 320c16ba3a57a3021289641a7ea2f382a486c3d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Thu, 11 Jan 2024 10:48:34 +0100 Subject: [PATCH 18/24] Set Ubuntu in combine-pages --- .github/workflows/regression-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index ae53b422..449add45 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -139,6 +139,7 @@ jobs: if-no-files-found: ignore combine-pages: + runs-on: ubuntu-latest # Add a dependency to the build job needs: [Jupyter, Pytest] steps: From 03c5b45f352e0906176470f236ce0bf6cf282c89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Thu, 11 Jan 2024 10:58:40 +0100 Subject: [PATCH 19/24] untar artifacts, upload for github pages --- .github/workflows/regression-tests.yml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 449add45..de65cf8c 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -130,13 +130,6 @@ jobs: name: tutorials path: ./doc/html if-no-files-found: ignore - - name: Upload Jupyter book for pages - # This is not a normal artifact but one that can be deployed to the GitHub pages in the next step - uses: actions/upload-pages-artifact@v3 - with: - name: github-pages # This name may not be changed according to the documentation - path: ./doc/html - if-no-files-found: ignore combine-pages: runs-on: ubuntu-latest @@ -145,13 +138,25 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - merge-multiple: true + merge-multiple: false - name: See what we've got - run: ls -la + run: | + ls -lah + tar -xf artifact.tar + ls -lah + - name: Upload subdirectories for pages + # This is not a normal artifact but one that can be deployed to the GitHub pages in the next step + uses: actions/upload-pages-artifact@v3 + with: + name: github-pages # This name may not be changed according to the documentation + path: | + ./tutorials + ./coverage + if-no-files-found: ignore deploy-pages: # Add a dependency to the build job - needs: [Jupyter, Pytest] + needs: combine-pages # Grant GITHUB_TOKEN the permissions required to make a Pages deployment permissions: pages: write # to deploy to Pages From 4dd44ca8df192639909405405c9f2e33c1ffaa95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Thu, 11 Jan 2024 11:02:34 +0100 Subject: [PATCH 20/24] Merge multiple artifacts --- .github/workflows/regression-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index de65cf8c..f63b0def 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -138,7 +138,7 @@ jobs: steps: - uses: actions/download-artifact@v4 with: - merge-multiple: false + merge-multiple: true - name: See what we've got run: | ls -lah From 5919f8cd0cc40498a68deefbf39d84eb2f798519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Thu, 11 Jan 2024 11:11:06 +0100 Subject: [PATCH 21/24] Switch to upload-artifact@v4 --- .github/workflows/regression-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index f63b0def..cec79f85 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -92,7 +92,7 @@ jobs: coverage xml -o coverage.xml coverage html --directory ./coverage/coverage - name: Upload test restults and coverage as an artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test results and coverage path: | @@ -125,7 +125,7 @@ jobs: mkdir ./doc/html mv ./doc/tutorials/_build/html ./doc/html/tutorials - name: Upload Jupyter book as an artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: tutorials path: ./doc/html @@ -141,9 +141,9 @@ jobs: merge-multiple: true - name: See what we've got run: | - ls -lah - tar -xf artifact.tar - ls -lah + ls -la + #tar -xf artifact.tar + #ls -la - name: Upload subdirectories for pages # This is not a normal artifact but one that can be deployed to the GitHub pages in the next step uses: actions/upload-pages-artifact@v3 From 25c14a59ae57102e84ab60614cdc5ed8c168bedd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Thu, 11 Jan 2024 11:20:39 +0100 Subject: [PATCH 22/24] upload only one path in combine-pages --- .github/workflows/regression-tests.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index cec79f85..4faf508a 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -142,16 +142,15 @@ jobs: - name: See what we've got run: | ls -la - #tar -xf artifact.tar - #ls -la - - name: Upload subdirectories for pages + mkdir pages + mv ./tutorials ./pages/tutorials + mv ./coverage ./pages/coverage + - name: Upload artifact for pages # This is not a normal artifact but one that can be deployed to the GitHub pages in the next step uses: actions/upload-pages-artifact@v3 with: - name: github-pages # This name may not be changed according to the documentation - path: | - ./tutorials - ./coverage + name: github-pages # This name may not be changed according to the documentation + path: ./pages # There must be only one path if-no-files-found: ignore deploy-pages: From dbffecc1832900d1ab1da42dc3221bf0b6c4a5a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Thu, 11 Jan 2024 11:26:19 +0100 Subject: [PATCH 23/24] HTML coverage: reduce folder by one level --- .github/workflows/regression-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 4faf508a..3dc3bb33 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -90,7 +90,7 @@ jobs: # Create some reports coverage report coverage xml -o coverage.xml - coverage html --directory ./coverage/coverage + coverage html --directory ./coverage - name: Upload test restults and coverage as an artifact uses: actions/upload-artifact@v4 with: @@ -139,7 +139,7 @@ jobs: - uses: actions/download-artifact@v4 with: merge-multiple: true - - name: See what we've got + - name: See what we've got and merge artifacts run: | ls -la mkdir pages From a39934c2dfe44237197cfe49502da2d8cda1145b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arne=20Vo=C3=9F?= Date: Thu, 11 Jan 2024 11:32:25 +0100 Subject: [PATCH 24/24] Split workflows, documentation --- .github/workflows/build.yml | 66 ++++++++++++++++++++++++++ .github/workflows/regression-tests.yml | 60 ++--------------------- 2 files changed, 69 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..1dd8613e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,66 @@ +# This workflow will try to build and install the software in different ways. + +name: Build and installation tests + +on: + push: + branches: ['master', 'devel'] + pull_request: + branches: '*' + +jobs: + pure-pip-installation: + # This stage only tests if the installation is possible. + # The evironment created herein will be discared and re-created in the test stage. + runs-on: ubuntu-latest + strategy: + matrix: + # Add multiple Python versions here to run tests on new(er) versions. + python-version: ["3.10"] + fail-fast: false + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Build and install + run: | + python -m pip install --upgrade pip + # Install with -e (in editable mode) to allow the tracking of the test coverage + pip install -e . + # Check result of installation + python -c "import loadskernel" + which loads-kernel + which model-viewer + which loads-compare + + conda-and-pip-installation: + # This stage only tests if the installation is possible. + # The evironment created herein will be discared and re-created in the test stage. + runs-on: ubuntu-latest + strategy: + matrix: + # Add multiple Python versions here to run tests on new(er) versions. + python-version: ["3.10", "3.11"] + fail-fast: false + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Build and install + run: | + # Install same requirements as to be used during regression testing + source $CONDA/etc/profile.d/conda.sh + conda activate + conda install -y -c conda-forge --file ./tests/requirements.txt + # Install with -e (in editable mode) to allow the tracking of the test coverage + pip install -e . + # Check result of installation + python -c "import loadskernel" + which loads-kernel + which model-viewer + which loads-compare + \ No newline at end of file diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml index 3dc3bb33..78ba9d82 100644 --- a/.github/workflows/regression-tests.yml +++ b/.github/workflows/regression-tests.yml @@ -1,6 +1,4 @@ -# This workflow will install and then lint the code with Flake8 and Pylint. -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - +# This workflow will run some regression tests. name: Regression Tests on: @@ -10,60 +8,6 @@ on: branches: '*' jobs: - pure-pip-installation: - # This stage only tests if the installation is possible. - # The evironment created herein will be discared and re-created in the test stage. - runs-on: ubuntu-latest - strategy: - matrix: - # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.10"] - fail-fast: false - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Build and install - run: | - python -m pip install --upgrade pip - # Install with -e (in editable mode) to allow the tracking of the test coverage - pip install -e . - # Check result of installation - python -c "import loadskernel" - which loads-kernel - which model-viewer - which loads-compare - - conda-and-pip-installation: - # This stage only tests if the installation is possible. - # The evironment created herein will be discared and re-created in the test stage. - runs-on: ubuntu-latest - strategy: - matrix: - # Add multiple Python versions here to run tests on new(er) versions. - python-version: ["3.10", "3.11"] - fail-fast: false - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Build and install - run: | - # Install same requirements as to be used during regression testing - source $CONDA/etc/profile.d/conda.sh - conda activate - conda install -y -c conda-forge --file ./tests/requirements.txt - # Install with -e (in editable mode) to allow the tracking of the test coverage - pip install -e . - # Check result of installation - python -c "import loadskernel" - which loads-kernel - which model-viewer - which loads-compare Pytest: runs-on: ubuntu-latest @@ -102,6 +46,8 @@ jobs: if-no-files-found: ignore Jupyter: + # Building the Jupyter book is not really a regression test. However, it has to be in this workflow due to the handling of + # the artifacts. runs-on: ubuntu-latest strategy: matrix: