Merge pull request #249 from SUNET/acrn-docker

manifest for platform-ops docker registries

files/docker_registry2/clean_registry.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+import sys
+import subprocess
+import ipaddress
+import yaml
+import registry
+
+# Nagios plugin exit status codes
+STATUS = {'OK': 0,
+          'WARNING': 1,
+          'CRITICAL': 2,
+          'UNKNOWN': 3,
+         }
+
+def get_container_ip(container_name):
+    process = subprocess.Popen([
+        '/usr/bin/docker', 'inspect',
+        '--format={{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}',
+        container_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
+    returncode = process.wait()
+
+    if returncode != 0:
+        print("Could not get IP address of the registry container")
+        sys.exit(STATUS['CRITICAL'])
+
+    # Read the output and remove \n
+    container_ip = process.stdout.read().rstrip()
+
+    # Convert the bytes into a string
+    container_ip = container_ip.decode("UTF-8")
+
+    try:
+        container_ip = ipaddress.ip_address(container_ip)
+    except ValueError:
+        print("Did not get a properly formatted IP address for the registry container")
+        sys.exit(STATUS['CRITICAL'])
+
+    return str(container_ip)
+
+
+def get_unmanaged_images(client, config):
+    all_images = set(client.list_images())
+    exact_images = set(config["exact_images"].keys())
+    group_images = set(registry.keep_images_like(all_images, config["group_images"]))
+    unmanaged_images = all_images - group_images - exact_images
+    return list(unmanaged_images)
+
+
+def clean(config):
+
+    if config['registry_url']:
+        registry_url = config['registry_url']
+    else:
+        registry_url = "http://{}:{}".format(get_container_ip(config['registry_container_name']),
+                                             config['registry_container_port'])
+    dry_run_arg = ["--dry-run"] if config.get("dry_run") else []
+
+    for image_to_clean in config['exact_images']:
+        print("Starting cleanup of image: {}".format(image_to_clean))
+        image_config = config["exact_images"][image_to_clean]
+        registry.main_loop(registry.parse_args([
+            "--delete", "--order-by-date", "--host", registry_url,
+            "--image", image_to_clean,
+            "--keep-tags-like", image_config["regex_of_tags_to_save"],
+            "--tags-like", image_config["regex_of_tags_to_delete"] or '.',
+            "--num", str(image_config["number_of_latest_builds_to_save"])]
+            + dry_run_arg))
+
+    for group in config['group_images']:
+        print("Starting cleanup of group: {}".format(group))
+        group_config = config["group_images"][group]
+        registry.main_loop(registry.parse_args([
+            "--delete", "--order-by-date", "--host", registry_url,
+            "--images-like", group,
+            "--keep-tags-like", group_config["regex_of_tags_to_save"],
+            "--tags-like", group_config["regex_of_tags_to_delete"] or '.',
+            "--num", str(group_config["number_of_latest_builds_to_save"])]
+            + dry_run_arg))
+
+    client = registry.Registry.create(registry_url, None, False)
+    unmanaged_images = get_unmanaged_images(client, config)
+    for image in unmanaged_images:
+        print(f"Starting cleanup of image: {image}")
+        registry.main_loop(
+            registry.parse_args(
+                [
+                    "--delete",
+                    "--order-by-date",
+                    "--host",
+                    registry_url,
+                    "--image",
+                    image,
+                    "--keep-by-hours",
+                    f"{24 * 7}",
+                ]
+                + dry_run_arg
+            )
+        )
+
+def parse_config(config_file):
+    with open(config_file, 'r') as conf_file:
+        config = yaml.safe_load(conf_file)
+
+    return config
+
+def main():
+
+    if len(sys.argv) != 2:
+        print('Usage: clean_registry.py config_file.yaml')
+        sys.exit(STATUS['WARNING'])
+
+    config_file = sys.argv[1]
+
+    # Remove arguments since otherwise they are read by
+    # clean_old_versions.py that doesn't recognize our arguments.
+    sys.argv = [sys.argv[0]]
+
+    config = parse_config(config_file)
+    clean(config)
+    sys.exit(STATUS['OK'])
+
+if __name__ == "__main__":
+    main()

files/docker_registry2/config_file.yaml

@@ -0,0 +1,20 @@
+# Specify a registry_url if the location of the registry is known,
+# otherwise specify a container name and port.
+registry_url:
+dry_run: true
+registry_data_dir: /var/lib/registry/docker/registry/v2
+registry_container_name: registry
+registry_container_port: 5000
+delete_docker_registry_image_path: /usr/local/bin/clean-registry/delete_docker_registry_image.py
+
+exact_images:
+    example:
+        regex_of_tags_to_save: 'latest$'
+        regex_of_tags_to_delete:
+        number_of_latest_builds_to_save: 5
+
+group_images:
+    '^example$':
+        regex_of_tags_to_save: 'latest$'
+        regex_of_tags_to_delete:
+        number_of_latest_builds_to_save: 5

files/docker_registry2/docker/docker-registry-auth/Dockerfile

@@ -0,0 +1,19 @@
+FROM debian:bullseye
+
+# Copying the Dockerfile to the image as documentation
+COPY Dockerfile /
+
+COPY registry-auth-ssl.conf /etc/apache2/sites-available/registry-auth-ssl.conf
+COPY setup.sh /opt/sunet/setup.sh
+COPY start.sh /start.sh
+RUN /opt/sunet/setup.sh
+
+WORKDIR /
+
+EXPOSE 443
+
+ENV SERVER_NAME docker.example.com
+ENV SSLVerifyDepth 1
+ENV PROXY_TARGET http://registry:5000
+
+CMD ["bash", "/start.sh"]

files/docker_registry2/docker/docker-registry-auth/README.md

@@ -0,0 +1,6 @@
+Docker Registry TLS Client Auth
+============
+
+* unauthenticated GET
+* authenticate all other operations
+* hope for the best

files/docker_registry2/docker/docker-registry-auth/registry-auth-ssl.conf

@@ -0,0 +1,79 @@
+ServerName ${SERVER_NAME}
+
+# Write to STDERR
+ErrorLog /proc/self/fd/2
+
+<VirtualHost *:443>
+    ServerName ${SERVER_NAME}
+    SSLEngine On
+    SSLProtocol             all -SSLv3 -TLSv1 -TLSv1.1
+    SSLCipherSuite          ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
+    SSLHonorCipherOrder     on
+    SSLCompression          off
+    SSLSessionTickets       off
+
+    SSLCertificateFile /etc/ssl/certs/${SERVER_NAME}.crt
+    SSLCertificateChainFile /etc/ssl/certs/${SERVER_NAME}-chain.crt
+    SSLCertificateKeyFile /etc/ssl/private/${SERVER_NAME}.key
+
+    # Everyone who wants to POST something have to present a client cert
+    # that is signed by this CA.
+    SSLCACertificateFile /etc/ssl/certs/${SERVER_NAME}-client-ca.crt
+    SSLVerifyClient optional
+    SSLVerifyDepth ${SSLVerifyDepth}
+    DocumentRoot /var/www/
+
+    ServerAdmin [email protected]
+
+    Header set Host "${SERVER_NAME}"
+    RequestHeader set X-Forwarded-Proto "https"
+
+    # HSTS (mod_headers is required) (15768000 seconds = 6 months)
+    Header always set Strict-Transport-Security "max-age=15768000"
+
+    ProxyRequests On
+
+    AddDefaultCharset utf-8
+
+    # Write to STDOUT
+    CustomLog /proc/self/fd/1 combined
+
+    LogLevel warn
+    ServerSignature off
+
+    AddDefaultCharset utf-8
+
+    ProxyPreserveHost  On
+    ProxyRequests      Off
+    SSLProxyEngine     On
+    ProxyPass          /v2  ${PROXY_TARGET}/v2
+    ProxyPassReverse   /v2  ${PROXY_TARGET}/v2
+
+    <Location />
+       Require all denied
+    </Location>
+
+    SSLUserName SSL_CLIENT_S_DN
+
+    <Location /v2>
+       Order deny,allow
+       Allow from all
+       <If "-f '/read-only'">
+          <RequireAll>
+             Require ssl
+             Require method GET
+          </RequireAll>
+       </If>
+       <Else>
+           <RequireAll>
+              Require ssl
+              Require expr ( "%{SSL_CLIENT_M_SERIAL}" != "35FE628DC47E4AF9" )
+              Require expr ( "%{SSL_CLIENT_M_SERIAL}" != "A2BB746766CEFDB9" )
+              <RequireAny>
+                 Require method GET
+                 Require ssl-verify-client
+              </RequireAny>
+          </RequireAll>
+       </Else>
+    </Location>
+</VirtualHost>

files/docker_registry2/docker/docker-registry-auth/setup.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+set -x
+
+export DEBIAN_FRONTEND noninteractive
+
+# Use the mirror hosted within SUNET in Sweden
+/bin/sed -i 's#deb.debian.org/debian$#ftp.se.debian.org/debian#' /etc/apt/sources.list
+
+# Update the image and install common tools for debugging
+# as well as packages needed for this image.
+apt-get update && \
+    apt-get -y dist-upgrade && \
+    apt-get install -y \
+      iputils-ping \
+      procps \
+      bind9-host \
+      netcat-openbsd \
+      apache2 \
+      ssl-cert \
+      augeas-tools \
+    && apt-get -y autoremove \
+    && apt-get autoclean
+
+# Do some more cleanup to save space
+rm -rf /var/lib/apt/lists/*
+
+# Remove default config
+rm /etc/apache2/sites-enabled/*
+
+# Enable required modules
+a2enmod rewrite
+a2enmod ssl
+a2enmod proxy
+a2enmod proxy_http
+a2enmod headers
+
+# Disable the status page
+a2dismod status
+
+# Enable the config we have specified
+a2ensite --maintmode registry-auth-ssl

files/docker_registry2/docker/docker-registry-auth/start.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+mkdir -p /var/lock/apache2 /var/run/apache2
+rm /var/run/apache2/apache2.pid
+
+exec env APACHE_LOCK_DIR=/var/lock/apache2 APACHE_RUN_DIR=/var/run/apache2 APACHE_PID_FILE=/var/run/apache2/apache2.pid APACHE_RUN_USER=www-data APACHE_RUN_GROUP=www-data APACHE_LOG_DIR=/var/log/apache2 apache2 -DFOREGROUND

files/docker_registry2/infra-ca.crt

@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIF8zCCA9ugAwIBAgIBADANBgkqhkiG9w0BAQsFADA/MSAwHgYDVQQDExdTVU5F
+VCBJbmZyYXN0cnVjdHVyZSBDQTEOMAwGA1UEChMFU1VORVQxCzAJBgNVBAYTAlNF
+MB4XDTE1MDMyNDIyMDA0M1oXDTI1MDMyMTIyMDA0M1owPzEgMB4GA1UEAxMXU1VO
+RVQgSW5mcmFzdHJ1Y3R1cmUgQ0ExDjAMBgNVBAoTBVNVTkVUMQswCQYDVQQGEwJT
+RTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANX8E3tAkO2lm7aU18ND
+hJtMARHObom9b+SpwrfgEI6dsnIqsrjzrZ1X+bv3AhlmWMS7aPr0BuvtsKxwcRaD
+TRdfM7ik7L40vXAkBwVWvXJvjdF5d+AZI750S5G1jSh/v8Nz+zHsai1mtdnx7FT6
+Pg1BJbwf0IyIHZClcnO/OmwElNnGVB5uNp3e/67KCqI4IhjAt+4G30mRfIpZ1KoU
+vexZsz++cZErCXEe0eWnhlnCjfobMKmEHhvX6RzvTbB80AL/tfrqnOEwD6y7iUOp
+N9FSTiHvHxRiD80WglLrh2qHzSn3it91RA1OvfY0HoIgdz1F/l07Nlm8a6WrrbRZ
+Pg+HzlZ31iy0/sqduj2fPrDuDDQn87Bu3ohsZPg1t700ZW+YMUWtmh9PHK04a2fI
+f9ET7llJPYzyOQ1apoiAgPRf4pnxOSOgjUhVDBY20ppTKxFJ7WY9JSKRPj92A6Ht
+2/uAfUapKPOPSaASIruVz7sZ7DqiWvq67uvRtwr5yytRoZ82HG1Z36DxSNUcJ2X8
+MmELT/ONQHolu8hiZCLDCienYWZUPBnaI9jblCqvmBrdlJzKdrWzb1zKEQNsducs
+Klwgh5hZ6tJLca3v/sDx7odUK4MF+vuhEyRZyXUQBZ3+m7iII+2mHLyZ2EUpfBjZ
+hlOERIttFErkPP5CsPkf8uvDAgMBAAGjgfkwgfYwHQYDVR0OBBYEFOcsnlEasB0B
+HeZCtCcaNZNwwG3XMB8GA1UdIwQYMBaAFOcsnlEasB0BHeZCtCcaNZNwwG3XMDsG
+CCsGAQUFBwEBBC8wLTArBggrBgEFBQcwAoYfaHR0cDovL2NhLnN1bmV0LnNlL2lu
+ZnJhL2NhLmNydDAxBgNVHR8EKjAoMCagJKAihiBodHRwOi8vY2Euc3VuZXQuc2Uv
+aW5mcmEvY3JsLnBlbTAjBgNVHRIEHDAahhhodHRwOi8vY2Euc3VuZXQuc2UvaW5m
+cmEwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQEL
+BQADggIBAHUlygRL3d5DEBKWVvsuWhWNq5O7QHqWYyRSEMbncHSsZJhryJvmI/4Z
+KI0UpBC6KBJDRGnKWnTfNUsNa6ZC/hPb+9RTdVV7ODq5T1xCp9bueVmf2x/CQEIK
+Rexwlv6+nMdUmFioxtTdKOCSkXu4L+dmIpzsbkUrl0wNSIeTga0StGyJZcbFq/cp
+qur89YaiDSZ490C7UrQSaMRmBYTqmISmtlLzpGEPR3e6xoJbxws3zKeUYfF4Fzzi
+t424jpgd+FHh7eEyNNqNqKP+kr/G4/BnJBzyr1uP+1/LSzJRHj/hNJV7R/8zr9KY
+hZxjP7YKLmRxfEaRIFcjDJOKEYzpN3MNWOWVKMduUEbk65sbTFIlY1wCDzV9rHeY
+81G82FQVmOMYc5RQI5ZcEqEUhOTv85bMF3rVpGR+tA8gfQWs0w8sa9wcEo/HfjXa
+wgu67cJe2grg9iaoh40cOUIbVFaHbkvOG3ZMJPOkye+nBuOJncWhpuxGRxgEvW/O
+gj5WnDwZ4J8hfGchaBSi5ZVEvUWpmx+NPzIp5YhHBRA5zadmd2fGIui/22fmJuDq
+syNaWN5Ncka6Ud5NSnuYJDZauC/3ftdwe5awkuQFon3qg0fiVprM+DOUNgakVGyF
+5G6c17lavZgC3xqdXYbnNkBTeaTgYYUdOxcT7WXARVw9ak5OhSw0
+-----END CERTIFICATE-----