From 4d56aeb727513263629d25cfb6d54238c67000bb Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Fri, 5 Apr 2024 15:56:28 -0400 Subject: [PATCH 1/5] [7.17] Fix Python systems tests with forked docker-compose package (#38743) * Manual port of docker CI fix from https://github.com/elastic/beats/pull/38199 * Fix order in requirements.txt. (cherry picked from commit 33b776a1b92188664944a49a176d1c80248671e3) # Conflicts: # libbeat/tests/system/requirements.txt # libbeat/tests/system/requirements_aix.txt # metricbeat/Dockerfile --- libbeat/tests/system/requirements.txt | 17 +++++++++++++++++ libbeat/tests/system/requirements_aix.txt | 17 +++++++++++++++++ metricbeat/Dockerfile | 15 ++++++++++++++- 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 8bdb021e8ec5..f0eaff978285 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -1,4 +1,11 @@ +<<<<<<< HEAD async-timeout==4.0.3 +======= +requests==2.31.0 +urllib3==1.26.18 +docker==6.1.3 +docker-compose @ git+https://github.com/pkoutsovasilis/compose@v1_fix +>>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) attrs==19.3.0 autopep8==1.5.4 backoff==2.2.1 @@ -11,10 +18,13 @@ chardet==3.0.4 charset-normalizer==3.3.2 cryptography==41.0.7 deepdiff==4.2.0 +<<<<<<< HEAD Deprecated==1.2.14 distro==1.9.0 docker==6.0.1 docker-compose==1.29.2 +======= +>>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -57,17 +67,24 @@ pytest-rerunfailures==9.1.1 pytest-timeout==1.4.2 python-dotenv==0.21.1 PyYAML==5.3.1 +<<<<<<< HEAD redis==4.4.4 requests==2.31.0 +======= +redis==2.10.6 +>>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) semver==2.8.1 six==1.15.0 stomp.py==4.1.22 termcolor==1.1.0 texttable==0.9.1 toml==0.10.1 +<<<<<<< HEAD tomli==2.0.1 typing_extensions==4.9.0 urllib3==1.26.18 +======= +>>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 diff --git a/libbeat/tests/system/requirements_aix.txt b/libbeat/tests/system/requirements_aix.txt index 8bdb021e8ec5..f0eaff978285 100644 --- a/libbeat/tests/system/requirements_aix.txt +++ b/libbeat/tests/system/requirements_aix.txt @@ -1,4 +1,11 @@ +<<<<<<< HEAD async-timeout==4.0.3 +======= +requests==2.31.0 +urllib3==1.26.18 +docker==6.1.3 +docker-compose @ git+https://github.com/pkoutsovasilis/compose@v1_fix +>>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) attrs==19.3.0 autopep8==1.5.4 backoff==2.2.1 @@ -11,10 +18,13 @@ chardet==3.0.4 charset-normalizer==3.3.2 cryptography==41.0.7 deepdiff==4.2.0 +<<<<<<< HEAD Deprecated==1.2.14 distro==1.9.0 docker==6.0.1 docker-compose==1.29.2 +======= +>>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -57,17 +67,24 @@ pytest-rerunfailures==9.1.1 pytest-timeout==1.4.2 python-dotenv==0.21.1 PyYAML==5.3.1 +<<<<<<< HEAD redis==4.4.4 requests==2.31.0 +======= +redis==2.10.6 +>>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) semver==2.8.1 six==1.15.0 stomp.py==4.1.22 termcolor==1.1.0 texttable==0.9.1 toml==0.10.1 +<<<<<<< HEAD tomli==2.0.1 typing_extensions==4.9.0 urllib3==1.26.18 +======= +>>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index b6da04173167..cec6d0d5821c 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,9 @@ +<<<<<<< HEAD FROM golang:1.21.8 +======= +FROM golang:1.21.9 +COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ +>>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) RUN \ apt update \ @@ -8,8 +13,13 @@ RUN \ python3-dev \ python3-pip \ python3-venv \ +<<<<<<< HEAD libaio-dev \ unzip \ +======= + libssl-dev \ + libffi-dev \ +>>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) && rm -rf /var/lib/apt/lists/* # Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts @@ -19,9 +29,12 @@ RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip3 install --upgrade pip==20.1.1 -RUN pip3 install --upgrade docker-compose==1.23.2 RUN pip3 install --upgrade setuptools==47.3.2 RUN pip3 install --upgrade PyYAML==5.3.1 +RUN pip3 install requests==2.31.0 +RUN pip3 install urllib3==1.26.18 +RUN pip3 install docker==6.1.3 +RUN pip3 install git+https://github.com/pkoutsovasilis/compose@v1_fix # Oracle instant client RUN cd /usr/lib \ From 358974663a089139a2e306f5ec26322bfb1da37d Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Fri, 5 Apr 2024 16:03:27 -0400 Subject: [PATCH 2/5] Resolve conflicts --- libbeat/tests/system/requirements.txt | 20 +------------------- libbeat/tests/system/requirements_aix.txt | 22 ---------------------- metricbeat/Dockerfile | 8 -------- 3 files changed, 1 insertion(+), 49 deletions(-) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index f0eaff978285..80b172556f95 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -1,11 +1,8 @@ -<<<<<<< HEAD -async-timeout==4.0.3 -======= requests==2.31.0 urllib3==1.26.18 docker==6.1.3 docker-compose @ git+https://github.com/pkoutsovasilis/compose@v1_fix ->>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) +async-timeout==4.0.3 attrs==19.3.0 autopep8==1.5.4 backoff==2.2.1 @@ -18,13 +15,6 @@ chardet==3.0.4 charset-normalizer==3.3.2 cryptography==41.0.7 deepdiff==4.2.0 -<<<<<<< HEAD -Deprecated==1.2.14 -distro==1.9.0 -docker==6.0.1 -docker-compose==1.29.2 -======= ->>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -67,24 +57,16 @@ pytest-rerunfailures==9.1.1 pytest-timeout==1.4.2 python-dotenv==0.21.1 PyYAML==5.3.1 -<<<<<<< HEAD redis==4.4.4 -requests==2.31.0 -======= -redis==2.10.6 ->>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) semver==2.8.1 six==1.15.0 stomp.py==4.1.22 termcolor==1.1.0 texttable==0.9.1 toml==0.10.1 -<<<<<<< HEAD tomli==2.0.1 typing_extensions==4.9.0 urllib3==1.26.18 -======= ->>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 diff --git a/libbeat/tests/system/requirements_aix.txt b/libbeat/tests/system/requirements_aix.txt index f0eaff978285..a747d3933bb9 100644 --- a/libbeat/tests/system/requirements_aix.txt +++ b/libbeat/tests/system/requirements_aix.txt @@ -1,11 +1,4 @@ -<<<<<<< HEAD async-timeout==4.0.3 -======= -requests==2.31.0 -urllib3==1.26.18 -docker==6.1.3 -docker-compose @ git+https://github.com/pkoutsovasilis/compose@v1_fix ->>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) attrs==19.3.0 autopep8==1.5.4 backoff==2.2.1 @@ -18,13 +11,6 @@ chardet==3.0.4 charset-normalizer==3.3.2 cryptography==41.0.7 deepdiff==4.2.0 -<<<<<<< HEAD -Deprecated==1.2.14 -distro==1.9.0 -docker==6.0.1 -docker-compose==1.29.2 -======= ->>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -67,24 +53,16 @@ pytest-rerunfailures==9.1.1 pytest-timeout==1.4.2 python-dotenv==0.21.1 PyYAML==5.3.1 -<<<<<<< HEAD redis==4.4.4 -requests==2.31.0 -======= -redis==2.10.6 ->>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) semver==2.8.1 six==1.15.0 stomp.py==4.1.22 termcolor==1.1.0 texttable==0.9.1 toml==0.10.1 -<<<<<<< HEAD tomli==2.0.1 typing_extensions==4.9.0 urllib3==1.26.18 -======= ->>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index cec6d0d5821c..fa1835cc9e10 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,9 +1,4 @@ -<<<<<<< HEAD FROM golang:1.21.8 -======= -FROM golang:1.21.9 -COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ ->>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) RUN \ apt update \ @@ -13,13 +8,10 @@ RUN \ python3-dev \ python3-pip \ python3-venv \ -<<<<<<< HEAD libaio-dev \ unzip \ -======= libssl-dev \ libffi-dev \ ->>>>>>> 33b776a1b9 ([7.17] Fix Python systems tests with forked docker-compose package (#38743)) && rm -rf /var/lib/apt/lists/* # Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts From 1d69a128a9ac124d6de66c0da6fe8b6c73e9a613 Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Fri, 5 Apr 2024 16:04:22 -0400 Subject: [PATCH 3/5] Restore uintentionally removed packages --- libbeat/tests/system/requirements.txt | 2 ++ libbeat/tests/system/requirements_aix.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 80b172556f95..7f70ff5d431e 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -15,6 +15,8 @@ chardet==3.0.4 charset-normalizer==3.3.2 cryptography==41.0.7 deepdiff==4.2.0 +Deprecated==1.2.14 +distro==1.9.0 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 diff --git a/libbeat/tests/system/requirements_aix.txt b/libbeat/tests/system/requirements_aix.txt index a747d3933bb9..02ab3f5efc18 100644 --- a/libbeat/tests/system/requirements_aix.txt +++ b/libbeat/tests/system/requirements_aix.txt @@ -11,6 +11,8 @@ chardet==3.0.4 charset-normalizer==3.3.2 cryptography==41.0.7 deepdiff==4.2.0 +Deprecated==1.2.14 +distro==1.9.0 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 From 9dcf9dbcad0cdc59790fe9fc8992e28f9d7cd53e Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Fri, 5 Apr 2024 16:04:54 -0400 Subject: [PATCH 4/5] Remove duplicate package. --- libbeat/tests/system/requirements.txt | 1 - libbeat/tests/system/requirements_aix.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 7f70ff5d431e..d249e904c5c2 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -68,7 +68,6 @@ texttable==0.9.1 toml==0.10.1 tomli==2.0.1 typing_extensions==4.9.0 -urllib3==1.26.18 wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 diff --git a/libbeat/tests/system/requirements_aix.txt b/libbeat/tests/system/requirements_aix.txt index 02ab3f5efc18..073269011ed0 100644 --- a/libbeat/tests/system/requirements_aix.txt +++ b/libbeat/tests/system/requirements_aix.txt @@ -64,7 +64,6 @@ texttable==0.9.1 toml==0.10.1 tomli==2.0.1 typing_extensions==4.9.0 -urllib3==1.26.18 wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 From d859a30e640a6f3d9b2d6aef2f5804aa706a819e Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Fri, 5 Apr 2024 17:54:15 -0400 Subject: [PATCH 5/5] Add dropped docker copy --- metricbeat/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index fa1835cc9e10..a16b942032d8 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,5 @@ FROM golang:1.21.8 +COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ RUN \ apt update \