Change to new Aventer Mesos URL (#4290)

* Change to new Aventer Mesos URL * Switch to Ubuntu-provided Docker to fix #4282 * Define a make load_docker to let you make the Docker image and load it into your daemon * Fix #4260 and also respect cgroups v2 * Fail to upgrade Singularity to one that can use cgroups v2 because libc upgrade breaks the compiler * Just update to Ubuntu 22.04 base * Improve Dockerfile comments * Fix Singularity expecting to be able to call itself from /usr/bin * Account for Singularity option change * Downgrade squashfs to one that ignores errors coming from not really being root * Keep squashfs-tools in main dependency list so its dependencies get installed
DataBiosphere · Dec 3, 2022 · 60a0e41 · 60a0e41
1 parent 04e13e9
commit 60a0e41
Show file tree

Hide file tree

Showing 6 changed files with 113 additions and 45 deletions.
diff --git a/Makefile b/Makefile
@@ -61,6 +61,11 @@ the appliance images to, for example:
 
 	TOIL_DOCKER_REGISTRY=quay.io/USER make docker
 
+You might also want to build just for one architecture and load into your
+Docker daemon. We have a 'load_docker' target for this.
+
+    make load_docker arch=amd64
+
 If Docker is not installed, Docker-related targets tasks and tests will be skipped. The
 same can be achieved by setting TOIL_DOCKER_REGISTRY to an empty string.
 
@@ -154,7 +159,7 @@ docker: toil_docker prometheus_docker grafana_docker mtail_docker
 
 pre_pull_docker:
 	# Pre-pull everything
-	for i in $$(seq 1 11); do if [[ $$i == "11" ]] ; then exit 1 ; fi ; docker pull ubuntu:20.04 && break || sleep 60; done
+	for i in $$(seq 1 11); do if [[ $$i == "11" ]] ; then exit 1 ; fi ; docker pull ubuntu:22.04 && break || sleep 60; done
 	for i in $$(seq 1 11); do if [[ $$i == "11" ]] ; then exit 1 ; fi ; docker pull prom/prometheus:v2.24.1 && break || sleep 60; done
 	for i in $$(seq 1 11); do if [[ $$i == "11" ]] ; then exit 1 ; fi ; docker pull grafana/grafana && break || sleep 60; done
 	for i in $$(seq 1 11); do if [[ $$i == "11" ]] ; then exit 1 ; fi ; docker pull sscaling/mtail && break || sleep 60; done
@@ -196,9 +201,15 @@ push_docker: docker
 	cd dashboard/grafana ; for i in $$(seq 1 6); do if [[ $$i == "6" ]] ; then exit 1 ; fi ; docker buildx build --platform $(arch) --push --tag=$(grafana_image):$(TOIL_DOCKER_TAG) -f Dockerfile . && break || sleep 60; done
 	cd dashboard/mtail ; for i in $$(seq 1 6); do if [[ $$i == "6" ]] ; then exit 1 ; fi ; docker buildx build --platform $(arch) --push --tag=$(mtail_image):$(TOIL_DOCKER_TAG) -f Dockerfile . && break || sleep 60; done
 
+load_docker: docker
+	cd docker ; docker buildx build --platform $(arch) --load --tag=$(docker_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
+	cd dashboard/prometheus ; docker buildx build --platform $(arch) --load --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) -f Dockerfile . 
+	cd dashboard/grafana ; docker buildx build --platform $(arch) --load --tag=$(grafana_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
+	cd dashboard/mtail ; docker buildx build --platform $(arch) --load --tag=$(mtail_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
+
 else
 
-docker docker_push clean_docker:
+docker push_docker load_docker clean_docker:
 	@printf "$(cyan)Skipping '$@' target as TOIL_DOCKER_REGISTRY is empty or Docker is not installed.$(normal)\n"
 
 endif

diff --git a/docker/Dockerfile.py b/docker/Dockerfile.py
@@ -53,6 +53,7 @@
                          'less',
                          'vim',
                          'git',
+                         'docker.io',
                          'time',
                          # Dependencies for Mesos which the deb doesn't actually list
                          'libsvn1',
@@ -83,7 +84,7 @@ def heredoc(s):
 motd = ''.join(l + '\\n\\\n' for l in motd.splitlines())
 
 print(heredoc('''
-    FROM ubuntu:20.04
+    FROM ubuntu:22.04
 
     ARG TARGETARCH
 
@@ -102,12 +103,13 @@ def heredoc(s):
 
     # Find a repo with a Mesos build.
     # See https://rpm.aventer.biz/README.txt
-    # A working snapshot is https://ipfs.io/ipfs/QmfTy9sXhHsgyWwosCJDfYR4fChTosA8HhoaMgmeJ5LSmS/
+    # A working snapshot is https://ipfs.io/ipfs/QmfTy9sXhHsgyWwosCJDfYR4fChTosA8HhoaMgmeJ5LSmS/ for https://rpm.aventer.biz/Ubuntu
+    # And one that works with https://rpm.aventer.biz/Ubuntu/focal (the new URL) is at https://ipfs.io/ipfs/Qmcrmx7T1YkEnyexMXdd7QjoBZxf7DMDrQ5ErUKi9mDRw6/
     # As archived with:
     # mkdir mesos-repo && cd mesos-repo
     # wget --recursive --restrict-file-names=windows -k --convert-links --no-parent --page-requisites https://rpm.aventer.biz/Ubuntu/ https://www.aventer.biz/assets/support_aventer.asc https://rpm.aventer.biz/README.txt
     # ipfs add -r .
-    RUN echo "deb https://rpm.aventer.biz/Ubuntu focal main" \
+    RUN echo "deb https://rpm.aventer.biz/Ubuntu/focal focal main" \
         > /etc/apt/sources.list.d/mesos.list \
         && curl https://www.aventer.biz/assets/support_aventer.asc | apt-key add -
 
@@ -119,10 +121,32 @@ def heredoc(s):
         rm -rf /var/lib/apt/lists/*
     
     # Install a particular old Debian Sid Singularity from somewhere.
-    ADD singularity-sources.tsv /etc/singularity/singularity-sources.tsv
-    RUN wget -q "$(cat /etc/singularity/singularity-sources.tsv | grep "^$TARGETARCH" | cut -f3)" && \
-        dpkg -i singularity-container_3*.deb && \
-        rm singularity-container_3*.deb && \
+    # It's 3.10, which is new enough to use cgroups2, but it needs a newer libc
+    # than Ubuntu 20.04 ships. So we need a 22.04+ base.
+    #
+    # But 22.04 ships squashfs-tools 4.4 or 4.5 or so, which is new enough that
+    # errors encountered during extraction produce a nonzero exit code, without
+    # a special option:
+    # <https://github.com/plougher/squashfs-tools/commit/1dd7f32e79b7600d379a4f26fb8d138ebdfc70be>.
+    # If unsquashfs thinks it is root, but it can't change UIDs and GIDs freely, it
+    # will continue but fail the whole command instead of returning success. It complains:
+    #
+    # set_attributes: failed to change uid and gids on /image/rootfs/etc/gshadow, because Invalid argument
+    #
+    # But inside a Kubernetes container we can be root but not actually be
+    # allowed to set UIDs and GIDs arbitrarily. Singularity can't handle this,
+    # and can't pass the flag to ignore these errors, and we can't wrap
+    # unsquashfs with a shell script because of how it gets mounted into the
+    # container under construction along with its libraries (see
+    # <https://github.com/apptainer/singularity/issues/6113#issuecomment-901897566>).
+    #
+    # So we need to make sure to install a downgraded squashfs first.
+    ADD extra-debs.tsv /etc/singularity/extra-debs.tsv
+    RUN wget -q "$(cat /etc/singularity/extra-debs.tsv | grep "^squashfs-tools.$TARGETARCH" | cut -f4)" && \
+        dpkg -i squashfs-tools_*.deb && \
+        wget -q "$(cat /etc/singularity/extra-debs.tsv | grep "^singularity-container.$TARGETARCH" | cut -f4)" && \
+        dpkg -i singularity-container_*.deb && \
+        rm singularity-container_*.deb && \
         sed -i 's!bind path = /etc/localtime!#bind path = /etc/localtime!g' /etc/singularity/singularity.conf && \
         mkdir -p /usr/local/libexec/toil && \
         mv /usr/bin/singularity /usr/local/libexec/toil/singularity-real \
@@ -135,9 +159,11 @@ def heredoc(s):
 
     ADD customDockerInit.sh /usr/bin/customDockerInit.sh
 
-    ADD singularity-wrapper.sh /usr/local/bin/singularity
+    # Wrap Singularity to use a Docker mirror instead of always Docker Hub
+    # We need to put it where the installed singularity expects singularity to actually be.
+    ADD singularity-wrapper.sh /usr/bin/singularity
 
-    RUN chmod 777 /usr/bin/waitForKey.sh && chmod 777 /usr/bin/customDockerInit.sh && chmod 777 /usr/local/bin/singularity
+    RUN chmod 777 /usr/bin/waitForKey.sh && chmod 777 /usr/bin/customDockerInit.sh && chmod 777 /usr/bin/singularity
 
     # The stock pip is too old and can't install from sdist with extras
     RUN curl -sS https://bootstrap.pypa.io/get-pip.py | {python}
@@ -153,12 +179,6 @@ def heredoc(s):
         && /home/s3am/bin/pip install s3am==2.0 \
         && ln -s /home/s3am/bin/s3am /usr/local/bin/
 
-    # Install statically linked version of docker client
-    RUN curl https://download.docker.com/linux/static/stable/$(if [ $TARGETARCH = amd64 ] ; then echo x86_64 ; else echo aarch64 ; fi)/docker-18.06.1-ce.tgz \
-        | tar -xvzf - --transform='s,[^/]*/,,g' -C /usr/local/bin/ \
-        && chmod u+x /usr/local/bin/docker \
-        && /usr/local/bin/docker -v
-
     # Fix for https://issues.apache.org/jira/browse/MESOS-3793
     ENV MESOS_LAUNCHER=posix
 

diff --git a/docker/extra-debs.tsv b/docker/extra-debs.tsv
@@ -0,0 +1,5 @@
+#Package	Architecture	SHA256	URL	URL	...
+squashfs-tools	amd64	883f4f4de16bd6a1551a7ef43cdb58f9f103fdc7f68b480bb24c60d569a463ae	https://snapshot.debian.org/archive/debian/20150805T214000Z/pool/main/s/squashfs-tools/squashfs-tools_4.2%2B20130409-2.1_amd64.deb	https://ipfs.io/ipfs/QmddmW7WSiTQjzEb3CsdqM8pjqX5X2sgud97agnHxyccE9/singularity-container/singularity-container_3.10.3+ds1-1_amd64.deb
+squashfs-tools	arm64	1d2e9e9dc8428c13e7af967f7f0984c35fd136b33e81a9216e80fdb6a8f562f1	https://snapshot.debian.org/archive/debian/20150805T214000Z/pool/main/s/squashfs-tools/squashfs-tools_4.2%2B20130409-2.1_arm64.deb	https://ipfs.io/ipfs/QmddmW7WSiTQjzEb3CsdqM8pjqX5X2sgud97agnHxyccE9/singularity-container/singularity-container_3.10.3+ds1-1_arm64.deb
+singularity-container	amd64	1e8b5aed949cba5d9ace3f242323a4b25f224027026d6a5f598f18ccf5dfc79c	https://snapshot.debian.org/archive/debian/20221013T214821Z/pool/main/s/singularity-container/singularity-container_3.10.3%2Bds1-1_amd64.deb	https://ipfs.io/ipfs/QmddmW7WSiTQjzEb3CsdqM8pjqX5X2sgud97agnHxyccE9/singularity-container/singularity-container_3.10.3+ds1-1_amd64.deb
+singularity-container	arm64	b439a7c4f6d8db7d4dcd2f0169e5894a659b842c26fccad8f411b2b748678353	https://snapshot.debian.org/archive/debian/20221013T214821Z/pool/main/s/singularity-container/singularity-container_3.10.3%2Bds1-1_arm64.deb	https://ipfs.io/ipfs/QmddmW7WSiTQjzEb3CsdqM8pjqX5X2sgud97agnHxyccE9/singularity-container/singularity-container_3.10.3+ds1-1_arm64.deb
diff --git a/docker/singularity-sources.tsv b/docker/singularity-sources.tsv
diff --git a/docker/singularity-wrapper.sh b/docker/singularity-wrapper.sh
@@ -56,8 +56,8 @@ if [[ "${ARGC}" -ge "2" && "${ARGV[1]}" == "pull" && ! -z "${MIRROR_HOST}" ]] ;
     HTTP_ARG=""
 
     if [[ "${REPLACED}" == "1" && "${MIRROR_HTTP}" == "1" ]] ; then
-        # We need to use HTTP and not HTTPS for the mirror, so we need to isnert the argument
-        HTTP_ARG="--nohttps"
+        # We need to use HTTP and not HTTPS for the mirror, so we need to insert the argument
+        HTTP_ARG="--no-https"
     fi
 
     # Run the pull with our extra args, and then all the args starting at 2.

diff --git a/src/toil/lib/threading.py b/src/toil/lib/threading.py
@@ -25,7 +25,7 @@
 import threading
 import traceback
 from contextlib import contextmanager
-from typing import Any, Dict, Iterator, Optional
+from typing import Any, Dict, Iterator, Optional, Union, cast
 
 import psutil  # type: ignore
 
@@ -84,7 +84,7 @@ def join(self, *args: Optional[float], **kwargs: Optional[float]) -> None:
             raise_(exc_type, exc_value, traceback)
 
 
-def cpu_count() -> Any:
+def cpu_count() -> int:
     """
     Get the rounded-up integer number of whole CPUs available.
 
@@ -105,42 +105,77 @@ def cpu_count() -> Any:
     cached = getattr(cpu_count, 'result', None)
     if cached is not None:
         # We already got a CPU count.
-        return cached
+        return cast(int, cached)
 
     # Get the fallback answer of all the CPUs on the machine
-    total_machine_size = psutil.cpu_count(logical=True)
+    total_machine_size = cast(int, psutil.cpu_count(logical=True))
 
     logger.debug('Total machine size: %d cores', total_machine_size)
 
-    try:
-        with open('/sys/fs/cgroup/cpu/cpu.cfs_quota_us') as stream:
-            # Read the quota
-            quota = int(stream.read())
-
-        logger.debug('CPU quota: %d', quota)
+    # cgroups may limit the size
+    cgroup_size: Union[float, int] = float('inf')
 
-        if quota == -1:
-            # Assume we can use the whole machine
-            return total_machine_size
-
-        with open('/sys/fs/cgroup/cpu/cpu.cfs_period_us') as stream:
-            # Read the period in which we are allowed to burn the quota
-            period = int(stream.read())
+    try:
+        # See if we can fetch these and use them
+        quota: Optional[int] = None
+        period: Optional[int] = None
+
+        # CGroups v1 keeps quota and period separate
+        CGROUP1_QUOTA_FILE = '/sys/fs/cgroup/cpu/cpu.cfs_quota_us'
+        CGROUP1_PERIOD_FILE = '/sys/fs/cgroup/cpu/cpu.cfs_period_us'
+        # CGroups v2 keeps both in one file, space-separated, quota first
+        CGROUP2_COMBINED_FILE = '/sys/fs/cgroup/cpu.max'
+
+        if os.path.exists(CGROUP1_QUOTA_FILE) and os.path.exists(CGROUP1_PERIOD_FILE):
+            logger.debug('CPU quota and period available from cgroups v1')
+            with open(CGROUP1_QUOTA_FILE) as stream:
+                # Read the quota
+                quota = int(stream.read())
+
+            with open(CGROUP1_PERIOD_FILE) as stream:
+                # Read the period in which we are allowed to burn the quota
+                period = int(stream.read())
+        elif os.path.exists(CGROUP2_COMBINED_FILE):
+            logger.debug('CPU quota and period available from cgroups v2')
+            with open(CGROUP2_COMBINED_FILE) as stream:
+                # Read the quota and the period together
+                quota, period = [int(part) for part in stream.read().split(' ')]
+        else:
+            logger.debug('CPU quota/period not available from cgroups v1 or cgroups v2')
 
-        logger.debug('CPU quota period: %d', period)
+        if quota is not None and period is not None:
+            # We got a quota and a period.
+            logger.debug('CPU quota: %d period: %d', quota, period)
 
-        # The thread count is how many multiples of a wall clcok period we can burn in that period.
-        cgroup_size = int(math.ceil(float(quota)/float(period)))
+            if quota == -1:
+                # But the quota can be -1 for unset.
+                # Assume we can use the whole machine.
+                return total_machine_size
 
-        logger.debug('Cgroup size in cores: %d', cgroup_size)
+            # The thread count is how many multiples of a wall clock period we
+            # can burn in that period.
+            cgroup_size = int(math.ceil(float(quota)/float(period)))
 
+            logger.debug('Control group size in cores: %d', cgroup_size)
     except:
         # We can't actually read these cgroup fields. Maybe we are a mac or something.
         logger.debug('Could not inspect cgroup: %s', traceback.format_exc())
-        cgroup_size = float('inf') # type: ignore
+
+    # CPU affinity may limit the size
+    affinity_size: Union[float, int] = float('inf')
+    if hasattr(os, 'sched_getaffinity'):
+        try:
+            logger.debug('CPU affinity available')
+            affinity_size = len(os.sched_getaffinity(0))
+            logger.debug('CPU affinity is restricted to %d cores', affinity_size)
+        except:
+             # We can't actually read this even though it exists.
+            logger.debug('Could not inspect scheduling affinity: %s', traceback.format_exc())
+    else:
+        logger.debug('CPU affinity not available')
 
     # Return the smaller of the actual thread count and the cgroup's limit, minimum 1.
-    result = max(1, min(cgroup_size, total_machine_size))
+    result = cast(int, max(1, min(min(affinity_size, cgroup_size), total_machine_size)))
     logger.debug('cpu_count: %s', str(result))
     # Make sure to remember it for the next call
     setattr(cpu_count, 'result', result)