Dsegog 369 migrate to python 3.11

.github/workflows/ci.yml

@@ -10,21 +10,9 @@ jobs:
   tests:
     strategy:
       fail-fast: false
-      matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
-        exclude:
-          - os: ubuntu-22.04
-            python-version: "3.8"
-          - os: ubuntu-22.04
-            python-version: "3.9"
-          - os: ubuntu-20.04
-            python-version: "3.10"
-          - os: ubuntu-20.04
-            python-version: "3.11"
-    runs-on: ${{ matrix.os }}
-
-    name: Python ${{ matrix.python-version }} Tests
+    runs-on: ubuntu-22.04
+
+    name: Python 3.11 Tests
     steps:
       - name: Checkout OperationsGateway API
         uses: actions/checkout@v3
@@ -33,18 +21,18 @@ jobs:
       - name: Install python-ldap dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y libsasl2-dev python${{ matrix.python-version }}-dev libldap2-dev libssl-dev
+          sudo apt-get install -y libsasl2-dev python3.11-dev libldap2-dev libssl-dev
 
       # Setup Python and environment dependencies (via cache)
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: 3.11
       - name: Load Pip cache
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
+          key: ${{ runner.os }}-pip-3.11-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
       - name: Install Poetry & Nox
         run: pip install -r .github/ci_requirements.txt
 
@@ -56,7 +44,6 @@ jobs:
 
       # Used to install mongoimport when Ubuntu 22.04 is used, identified at https://github.com/actions/runner-images/issues/6626#issuecomment-1327744126
       - name: Install MongoDB Database Tools
-        if: matrix.os == 'ubuntu-22.04'
         run: |
           sudo apt-get update
           sudo apt-get install -y wget gnupg
@@ -74,7 +61,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}-poetry-3.11-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
       - name: Install dependencies
         run: poetry install --without simulated-data
 
@@ -133,13 +120,12 @@ jobs:
         uses: actions/cache@v3
         with:
           path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/tests*
-          key: ${{ runner.os }}-poetry-nox-tests-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}-poetry-nox-tests-3.11-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
 
       # Run Nox tests session, saves and uploads a coverage report to codecov
       - name: Run Nox tests session
-        run: nox -p ${{ matrix.python-version }} -s tests -- --cov=operationsgateway_api --cov-report=xml
+        run: nox -p 3.11 -s tests -- --cov=operationsgateway_api --cov-report=xml
       - name: Upload code coverage report
-        if: matrix.python-version == '3.8'
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
@@ -152,23 +138,23 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8.14"
+          python-version: "3.11"
       - name: Checkout OperationsGateway API
         uses: actions/checkout@v3
 
       - name: Load Pip cache
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
+          key: ${{ runner.os }}-pip-3.11-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
       - name: Install Poetry & Nox
         run: pip install -r .github/ci_requirements.txt
 
       - name: Load Poetry cache for Nox lint session
         uses: actions/cache@v3
         with:
           path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/lint*
-          key: ${{ runner.os }}-poetry-nox-lint-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}-poetry-nox-lint-3.11-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
 
       - name: Run Nox lint session
         run: nox -s lint
@@ -181,23 +167,23 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8.14"
+          python-version: "3.11"
       - name: Checkout OperationsGateway API
         uses: actions/checkout@v3
 
       - name: Load Pip cache
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
+          key: ${{ runner.os }}-pip-3.11-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
       - name: Install Poetry & Nox
         run: pip install -r .github/ci_requirements.txt
 
       - name: Load Poetry cache for Nox black session
         uses: actions/cache@v3
         with:
           path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/black*
-          key: ${{ runner.os }}-poetry-nox-black-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}-poetry-nox-black-3.11-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
 
       - name: Run Nox black session
         run: nox -s black -- --check --diff
@@ -210,23 +196,23 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8.14"
+          python-version: "3.11"
       - name: Checkout OperationsGateway API
         uses: actions/checkout@v3
 
       - name: Load Pip cache
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
+          key: ${{ runner.os }}-pip-3.11-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
       - name: Install Poetry & Nox
         run: pip install -r .github/ci_requirements.txt
 
       - name: Load Poetry cache for Nox safety session
         uses: actions/cache@v3
         with:
           path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/safety*
-          key: ${{ runner.os }}-poetry-nox-safety-${{ matrix.python-version }}-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}-poetry-nox-safety-3.11-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
 
       - name: Run Nox safety session
         run: nox -s safety

README.md

@@ -10,20 +10,20 @@ Assuming default configuration, the API will exist on 127.0.0.1:8000. You can vi
 
 
 ## Environment Setup
-If not already present, you may need to install development tools for the desired Python version using the appropriate package manager for your OS. For example, for Python3.8 on Fedora or RHEL:
+If not already present, you may need to install development tools for the desired Python version using the appropriate package manager for your OS. For example, for Python3.11 on Fedora or RHEL:
 ```bash
-sudo dnf install "@Development Tools" python38 python38-pip python38-setuptools python38-devel openldap-devel git
+sudo dnf install "@Development Tools" python3.11 python3.11-pip python3.11-setuptools python3.11-devel openldap-devel git
 ```
 
-[Poetry](https://python-poetry.org/) is used to manage the dependencies of this API. To install Poetry, follow [the instructions](https://python-poetry.org/docs/master/#installing-with-the-official-installer) from their documentation.
+[Poetry](https://python-poetry.org/) is used to manage the dependencies of this API. To install Poetry, follow [the instructions](https://python-poetry.org/docs/) from their documentation.
 
-To install the project's dependencies, execute `poetry install`. The dependencies and the code in this repo are compatible with Python >=3.8, <3.12.
+To install the project's dependencies, execute `poetry install`. The dependencies and the code in this repo are compatible with Python >=3.11, <3.12.
 
 ## Nox Sessions
 Like [DataGateway API](https://github.com/ral-facilities/datagateway-api), this repository contains a [Nox](https://nox.thea.codes) file (`noxfile.py`) which exists in the root level of this repository. There are a handful of sessions which help with repetitive tasks during development To install Nox, use the following command:
 
 ```bash
-pip install --user --upgrade nox
+python3.11 -m pip install --user --upgrade nox
 ```
 
 To run a specific Nox session, use the following:

docs/github_actions.md

@@ -1,9 +1,6 @@
 # GitHub Actions
 
-For our CI, we use GitHub Actions. There are multiple jobs which run the tests in multiple versions of Python (running an ingestion script beforehand to load data for the tests) and perform code formatting/linting on the codebase etc. The steps are fairly self-explanatory, but there are a few things worth noting that may not be immediately obvious.
-
-## Ubuntu 20.04 & 22.04
-The Python tests are across a variety of Python versions, some using Ubuntu 20.04 runners and others using Ubuntu 22.04. This is because the build dependencies for python-ldap (installed using `apt-get`) are specific to Python versions and the newer versions aren't available in Ubuntu 20.04. The solution to this is running tests across all desired Python versions and across both OSs, but disabling one of the operating systems from running on each version of Python. 22.04 is disabled on 3.8 & 3.9, 20.04 is disabled on 3.10 & 3.11.
+For our CI, we use GitHub Actions. The workflow runs the ingestion script beforehand to load data for the tests, and perform code formatting/linting on the codebase etc. The steps are fairly self-explanatory, but there are a few things worth noting that may not be immediately obvious.
 
 ## Extra Steps to Install mongoimport on 22.04 Runners
 A consequence of using Ubuntu 22.04 is the loss of command line tools from MongoDB from the runners' default environment. This is important as our scripts for ingesting test data into the API use `mongoimport` to import data stored in JSON files. To workaround this, the CI contains a step that runs on 22.04 which executes a series of `apt` commands to install these tools. The [pull request](https://github.com/ral-facilities/operationsgateway-api/pull/97) that made these changes provide a couple of references to discussions on other GitHub repos.
@@ -14,10 +11,4 @@ The CI used to use Echo to store data for the tests. However, this was determine
 To combat this issue, the CI now sets up a local instance of `minio` using a Docker container which the API connects to when it's launched in the ingestion script. Minio is a way of having S3 storage locally, mimicking AWS (and therefore Echo S3).
 
 ## Caching
-To speed up the workflows, caching is used to store Python dependencies and load them into the virtual environments. This is done for both Pip and poetry - loading the `pip` cache helps speed up the installation of Poetry and Nox and the `poetry` cache helps with the project dependencies. The caching is done in a fairly standard way, but the cache key includes something that keys from other repos might not include, `env.pythonLocation`.
-
-An issue was noticed when merging a pull request - tests on Python 3.9 & 3.10 failed on `main`, but had previously passed on the branch used for the PR. This was because the virtual environments for the Nox sessions of the tests was broken - `python` couldn't be found, causing the `pytest` command to not be found. After some investigation, it was noticed that the Python versions used was slightly different, specifically the patch versions. Someone has encountered a similar issue in the past (see [#735 on the Nox repo](https://github.com/wntrblm/nox/issues/735)).
-
-The problem was caused by the caching. The caches were created using a different patch version of Python (e.g. 3.10.13), but a slightly newer version was being used on that Actions run (e.g. 3.10.14). This can happen because the default Python versions on the runners can change as the images are updated. This happened with Ubuntu 22.04 in this case - [this PR](https://github.com/actions/runner-images/pull/9560) shows that Python 3.9 & 3.10 had patch version updates, causing this problem. Unless a patch version is specified, `setup-python` uses the default versions on the runner images.
-
-A fix has been applied to the cache key, as suggested [in this issue comment](https://github.com/wntrblm/nox/issues/735#issuecomment-1854599576). `env.pythonLocation` has been added to the key because this includes the patch version of the Python version being used. This means that different caches will be used for 3.10.13 and 3.10.14, preventing this issue from occuring in the future.
+To speed up the workflows, caching is used to store Python dependencies and load them into the virtual environments. This is done for both Pip and poetry - loading the `pip` cache helps speed up the installation of Poetry and Nox and the `poetry` cache helps with the project dependencies. The caching is done in a fairly standard way, but the cache key includes something that keys from other repos might not include, `env.pythonLocation`.

noxfile.py

@@ -110,7 +110,7 @@ def safety(session):
             session.log("Error: The temporary requirements file could not be closed")
 
 
-@nox.session(python=["3.8", "3.9", "3.10", "3.11"], reuse_venv=True)
+@nox.session(python=["3.11"], reuse_venv=True)
 def tests(session):
     args = session.posargs
     session.run("poetry", "install", "--without", "simulated-data", external=True)

operationsgateway_api/src/routes/records.py

@@ -163,11 +163,11 @@ async def count_records(
 async def convert_search_ranges(
     access_token: AuthoriseToken,
     shotnum_range: Json = Query(
-        {},
+        "{}",
         description='Min and max shot number range (e.g. {"min": 200, "max": 500})',
     ),
     date_range: Json = Query(
-        {},
+        "{}",
         description="From and to date range (e.g."
         ' {"from": "2022-04-07 14:16:19", "to": "2022-04-07 21:00:00"})',
     ),
@@ -196,7 +196,7 @@ async def get_record_by_id(
             description="`_id` of the record to fetch from the database",
         ),
     ],
-    conditions: Json = Query({}, description="Conditions to apply to the query"),
+    conditions: Json = Query("{}", description="Conditions to apply to the query"),
     truncate: Optional[bool] = Query(
         False,
         description="Parameter used for development to reduce the output of"