CI: package docker tests with uv (crowdsecurity#3429)

.github/workflows/docker-tests.yml

@@ -49,28 +49,30 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=min
 
-      - name: "Setup Python"
+      - name: "Create Docker network"
+        run: docker network create net-test
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: 0.5.24
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: "Set up Python"
         uses: actions/setup-python@v5
         with:
-          python-version: "3.x"
-          cache: 'pipenv'
+          python-version-file: "./docker/test/.python-version"
 
-      - name: "Install dependencies"
+      # running serially to reduce test flakiness
+      - name: Lint and run the tests
         run: |
           cd docker/test
-          python -m pip install --upgrade pipenv wheel
-          pipenv install --deploy
-
-      - name: "Create Docker network"
-        run: docker network create net-test
-
-      - name: "Run tests"
+          uv sync --all-extras --dev --locked
+          uv run ruff check
+          uv run pytest tests -n 1 --durations=0 --color=yes
         env:
           CROWDSEC_TEST_VERSION: test
           CROWDSEC_TEST_FLAVORS: ${{ matrix.flavor }}
           CROWDSEC_TEST_NETWORK: net-test
           CROWDSEC_TEST_TIMEOUT: 90
-        # running serially to reduce test flakiness
-        run: |
-          cd docker/test
-          pipenv run pytest -n 1 --durations=0 --color=yes

Dockerfile

@@ -1,5 +1,5 @@
 # vim: set ft=dockerfile:
-FROM golang:1.23-alpine3.20 AS build
+FROM docker.io/golang:1.23-alpine3.20 AS build
 
 ARG BUILD_VERSION
 
@@ -31,7 +31,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 CGO_CFLAGS="-D_LARGEFILE64_
     # In case we need to remove agents here..
     # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete
 
-FROM alpine:latest AS slim
+FROM docker.io/alpine:latest AS slim
 
 RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash rsync && \
     mkdir -p /staging/etc/crowdsec && \

Dockerfile.debian

@@ -1,5 +1,5 @@
 # vim: set ft=dockerfile:
-FROM golang:1.23-bookworm AS build
+FROM docker.io/golang:1.23-bookworm AS build
 
 ARG BUILD_VERSION
 
@@ -36,7 +36,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
     # In case we need to remove agents here..
     # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete
 
-FROM debian:bookworm-slim AS slim
+FROM docker.io/debian:bookworm-slim AS slim
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV DEBCONF_NOWARNINGS="yes"

docker/test/.python-version

@@ -0,0 +1 @@
+3.12

docker/test/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "crowdsec-docker-tests"
+version = "0.1.0"
+description = "Docker tests for Crowdsec"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "pytest>=8.3.4",
+    "pytest-cs",
+    "pytest-dotenv>=0.5.2",
+    "pytest-xdist>=3.6.1",
+]
+
+[dependency-groups]
+dev = [
+    "ipdb>=0.13.13",
+    "ruff>=0.9.3",
+]
+
+[tool.uv.sources]
+pytest-cs = { git = "https://github.com/crowdsecurity/pytest-cs" }
+
+[tool.ruff]
+
+line-length = 120
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "I",   # isort
+    "C",   # flake8-comprehensions
+    "B",   # flake8-bugbear
+    "UP",  # pyupgrade
+    "C90", # macabe
+]
+
+ignore = [
+    "B008", # do not perform function calls in argument defaults
+]

docker/test/tests/conftest.py

@@ -1,11 +1,6 @@
-
 pytest_plugins = ("cs",)
 
 
 def pytest_configure(config):
-    config.addinivalue_line(
-        'markers', 'docker: mark tests for lone or manually orchestrated containers'
-    )
-    config.addinivalue_line(
-        'markers', 'compose: mark tests for docker compose projects'
-    )
+    config.addinivalue_line("markers", "docker: mark tests for lone or manually orchestrated containers")
+    config.addinivalue_line("markers", "compose: mark tests for docker compose projects")

docker/test/tests/test_agent.py

@@ -10,12 +10,12 @@
 def test_no_agent(crowdsec, flavor):
     """Test DISABLE_AGENT=true"""
     env = {
-        'DISABLE_AGENT': 'true',
+        "DISABLE_AGENT": "true",
     }
     with crowdsec(flavor=flavor, environment=env) as cs:
         cs.wait_for_log("*CrowdSec Local API listening on *:8080*")
-        cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
-        res = cs.cont.exec_run('cscli lapi status')
+        cs.wait_for_http(8080, "/health", want_status=HTTPStatus.OK)
+        res = cs.cont.exec_run("cscli lapi status")
         assert res.exit_code == 0
         stdout = res.output.decode()
         assert "You can successfully interact with Local API (LAPI)" in stdout
@@ -24,64 +24,72 @@ def test_no_agent(crowdsec, flavor):
 def test_machine_register(crowdsec, flavor, tmp_path_factory):
     """A local agent is always registered for use by cscli"""
 
-    data_dir = tmp_path_factory.mktemp('data')
+    data_dir = tmp_path_factory.mktemp("data")
 
     env = {
-        'DISABLE_AGENT': 'true',
+        "DISABLE_AGENT": "true",
     }
 
     volumes = {
-        data_dir: {'bind': '/var/lib/crowdsec/data', 'mode': 'rw'},
+        data_dir: {"bind": "/var/lib/crowdsec/data", "mode": "rw"},
     }
 
     with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
-        cs.wait_for_log([
+        cs.wait_for_log(
+            [
                 "*Generate local agent credentials*",
                 "*CrowdSec Local API listening on *:8080*",
-            ])
-        cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
-        res = cs.cont.exec_run('cscli lapi status')
+            ]
+        )
+        cs.wait_for_http(8080, "/health", want_status=HTTPStatus.OK)
+        res = cs.cont.exec_run("cscli lapi status")
         assert res.exit_code == 0
         stdout = res.output.decode()
         assert "You can successfully interact with Local API (LAPI)" in stdout
 
     # The local agent is not registered, because we didn't persist local_api_credentials.yaml
 
     with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
-        cs.wait_for_log([
+        cs.wait_for_log(
+            [
                 "*Generate local agent credentials*",
                 "*CrowdSec Local API listening on *:8080*",
-            ])
-        cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
-        res = cs.cont.exec_run('cscli lapi status')
+            ]
+        )
+        cs.wait_for_http(8080, "/health", want_status=HTTPStatus.OK)
+        res = cs.cont.exec_run("cscli lapi status")
         assert res.exit_code == 0
         stdout = res.output.decode()
         assert "You can successfully interact with Local API (LAPI)" in stdout
 
-    config_dir = tmp_path_factory.mktemp('config')
+    config_dir = tmp_path_factory.mktemp("config")
 
-    volumes[config_dir] = {'bind': '/etc/crowdsec', 'mode': 'rw'}
+    volumes[config_dir] = {"bind": "/etc/crowdsec", "mode": "rw"}
 
     with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
-        cs.wait_for_log([
+        cs.wait_for_log(
+            [
                 "*Generate local agent credentials*",
                 "*CrowdSec Local API listening on *:8080*",
-            ])
-        cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
-        res = cs.cont.exec_run('cscli lapi status')
+            ]
+        )
+        cs.wait_for_http(8080, "/health", want_status=HTTPStatus.OK)
+        res = cs.cont.exec_run("cscli lapi status")
         assert res.exit_code == 0
         stdout = res.output.decode()
         assert "You can successfully interact with Local API (LAPI)" in stdout
 
     # The local agent is now already registered
 
     with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
-        cs.wait_for_log([
+        cs.wait_for_log(
+            [
                 "*Local agent already registered*",
                 "*CrowdSec Local API listening on *:8080*",
-            ])
-        cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
-        res = cs.cont.exec_run('cscli lapi status')
+            ]
+        )
+        cs.wait_for_http(8080, "/health", want_status=HTTPStatus.OK)
+        res = cs.cont.exec_run("cscli lapi status")
         assert res.exit_code == 0
         stdout = res.output.decode()
         assert "You can successfully interact with Local API (LAPI)" in stdout

docker/test/tests/test_agent_only.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
-from http import HTTPStatus
 import random
+from http import HTTPStatus
 
 import pytest
 
@@ -10,30 +10,30 @@
 
 def test_split_lapi_agent(crowdsec, flavor):
     rand = str(random.randint(0, 10000))
-    lapiname = f'lapi-{rand}'
-    agentname = f'agent-{rand}'
+    lapiname = f"lapi-{rand}"
+    agentname = f"agent-{rand}"
 
     lapi_env = {
-        'AGENT_USERNAME': 'testagent',
-        'AGENT_PASSWORD': 'testpassword',
+        "AGENT_USERNAME": "testagent",
+        "AGENT_PASSWORD": "testpassword",
     }
 
     agent_env = {
-        'AGENT_USERNAME': 'testagent',
-        'AGENT_PASSWORD': 'testpassword',
-        'DISABLE_LOCAL_API': 'true',
-        'LOCAL_API_URL': f'http://{lapiname}:8080',
+        "AGENT_USERNAME": "testagent",
+        "AGENT_PASSWORD": "testpassword",
+        "DISABLE_LOCAL_API": "true",
+        "LOCAL_API_URL": f"http://{lapiname}:8080",
     }
 
     cs_lapi = crowdsec(name=lapiname, environment=lapi_env, flavor=flavor)
     cs_agent = crowdsec(name=agentname, environment=agent_env, flavor=flavor)
 
     with cs_lapi as lapi:
         lapi.wait_for_log("*CrowdSec Local API listening on *:8080*")
-        lapi.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
+        lapi.wait_for_http(8080, "/health", want_status=HTTPStatus.OK)
         with cs_agent as agent:
             agent.wait_for_log("*Starting processing data*")
-            res = agent.cont.exec_run('cscli lapi status')
+            res = agent.cont.exec_run("cscli lapi status")
             assert res.exit_code == 0
             stdout = res.output.decode()
             assert "You can successfully interact with Local API (LAPI)" in stdout

docker/test/tests/test_bouncer.py

@@ -5,8 +5,8 @@
 """
 
 import hashlib
-from http import HTTPStatus
 import json
+from http import HTTPStatus
 
 import pytest
 
@@ -21,36 +21,33 @@ def hex512(s):
 def test_register_bouncer_env(crowdsec, flavor):
     """Test installing bouncers at startup, from envvar"""
 
-    env = {
-        'BOUNCER_KEY_bouncer1name': 'bouncer1key',
-        'BOUNCER_KEY_bouncer2name': 'bouncer2key'
-    }
+    env = {"BOUNCER_KEY_bouncer1name": "bouncer1key", "BOUNCER_KEY_bouncer2name": "bouncer2key"}
 
     with crowdsec(flavor=flavor, environment=env) as cs:
         cs.wait_for_log("*Starting processing data*")
-        cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
-        res = cs.cont.exec_run('cscli bouncers list -o json')
+        cs.wait_for_http(8080, "/health", want_status=HTTPStatus.OK)
+        res = cs.cont.exec_run("cscli bouncers list -o json")
         assert res.exit_code == 0
         j = json.loads(res.output)
         assert len(j) == 2
         bouncer1, bouncer2 = j
-        assert bouncer1['name'] == 'bouncer1name'
-        assert bouncer2['name'] == 'bouncer2name'
+        assert bouncer1["name"] == "bouncer1name"
+        assert bouncer2["name"] == "bouncer2name"
 
         # add a second bouncer at runtime
-        res = cs.cont.exec_run('cscli bouncers add bouncer3name -k bouncer3key')
+        res = cs.cont.exec_run("cscli bouncers add bouncer3name -k bouncer3key")
         assert res.exit_code == 0
-        res = cs.cont.exec_run('cscli bouncers list -o json')
+        res = cs.cont.exec_run("cscli bouncers list -o json")
         assert res.exit_code == 0
         j = json.loads(res.output)
         assert len(j) == 3
         bouncer3 = j[2]
-        assert bouncer3['name'] == 'bouncer3name'
+        assert bouncer3["name"] == "bouncer3name"
 
         # remove all bouncers
-        res = cs.cont.exec_run('cscli bouncers delete bouncer1name bouncer2name bouncer3name')
+        res = cs.cont.exec_run("cscli bouncers delete bouncer1name bouncer2name bouncer3name")
         assert res.exit_code == 0
-        res = cs.cont.exec_run('cscli bouncers list -o json')
+        res = cs.cont.exec_run("cscli bouncers list -o json")
         assert res.exit_code == 0
         j = json.loads(res.output)
         assert len(j) == 0