From 9c95a1502cb49b1d2e90f817f37b36ee783e568e Mon Sep 17 00:00:00 2001 From: Max Norton Date: Fri, 17 Jan 2025 18:03:41 +0000 Subject: [PATCH 1/2] Update Ansible deployment to allow for unprivileged deployment using Podman - change Ansible collection from community.docker to containers.podman - update all Ansible tasks to use podman instead of docker, we install podman-docker for a docker compatible CLI but we need to be podman specific in our Ansible playbook - move group_vars/reductionist -> group_vars/all so all "reductionist_" prefixed vars can be used across plays, specifically Step and Reductionist - update documentation --- deployment/group_vars/{reductionist => all} | 4 +- deployment/requirements.yml | 2 +- deployment/site.yml | 242 ++++++++++---------- docs/deployment.md | 43 ++-- 4 files changed, 150 insertions(+), 141 deletions(-) rename deployment/group_vars/{reductionist => all} (92%) diff --git a/deployment/group_vars/reductionist b/deployment/group_vars/all similarity index 92% rename from deployment/group_vars/reductionist rename to deployment/group_vars/all index 7e4ae30..82a37a7 100644 --- a/deployment/group_vars/reductionist +++ b/deployment/group_vars/all @@ -13,7 +13,7 @@ reductionist_image: "ghcr.io/stackhpc/reductionist-rs" reductionist_tag: "latest" # List of container networks. reductionist_networks: - - name: host + - host # Container environment. reductionist_env: OTEL_EXPORTER_JAEGER_AGENT_HOST: "{{ hostvars[(groups['jaeger'] | default([]) + [inventory_hostname])[0]].ansible_facts.default_ipv4.address }}" @@ -21,7 +21,7 @@ reductionist_env: REDUCTIONIST_HTTPS: "true" REDUCTIONIST_PORT: "8081" # Path to certificates directory on remote host. -reductionist_remote_certs_path: "{{ ansible_facts.env.HOME }}/.config/reductionist/certs" +reductionist_remote_certs_path: "{{ ansible_facts.env.HOME }}/certs" # Path to certificates directory in container. reductionist_container_certs_path: "/root/.config/reductionist/certs" # List of container volume mounts. diff --git a/deployment/requirements.yml b/deployment/requirements.yml index d1a4685..68b4a8e 100644 --- a/deployment/requirements.yml +++ b/deployment/requirements.yml @@ -1,3 +1,3 @@ --- collections: - - community.docker + - containers.podman diff --git a/deployment/site.yml b/deployment/site.yml index 79316fc..9e0e0ef 100644 --- a/deployment/site.yml +++ b/deployment/site.yml @@ -1,42 +1,27 @@ --- # See deployment/README.md for usage -- name: Install Docker - hosts: docker +- name: Install Podman Docker + hosts: podman tags: - - docker + - podman become: true tasks: - - name: Ensure docker is installed on Ubuntu + - name: Ensure podman docker is installed on Ubuntu when: ansible_facts["os_family"] | lower == "debian" ansible.builtin.package: - name: docker.io + name: + - passt + - podman-docker state: present update_cache: true - - name: Ensure docker repo signing key exists on RedHat - when: ansible_facts["os_family"] | lower == "redhat" - ansible.builtin.rpm_key: - key: "https://download.docker.com/linux/centos/gpg" - state: present - - - name: Ensure docker repo exists on RedHat - when: ansible_facts["os_family"] | lower == "redhat" - ansible.builtin.yum_repository: - name: docker - description: docker repository - baseurl: "https://download.docker.com/linux/centos/$releasever/$basearch/stable" - enabled: true - gpgcheck: true - gpgkey: "https://download.docker.com/linux/centos/gpg" - - - name: Ensure docker is installed on RedHat + - name: Ensure podman docker is installed on RedHat when: ansible_facts["os_family"] | lower == "redhat" ansible.builtin.package: name: - - docker-ce - - docker-ce-cli - - containerd.io + - passt + - podman-docker state: present update_cache: true @@ -47,22 +32,10 @@ - python3-pip state: present - - name: Check docker is running - ansible.builtin.service: - name: "docker" - enabled: true - state: started - - - name: Ensure docker python package is present - ansible.builtin.pip: - name: - - docker - - name: Deploy step CA hosts: step-ca tags: - step-ca - become: true tasks: - name: Assert that there is only one CA server ansible.builtin.assert: @@ -70,16 +43,15 @@ groups['step-ca'] | length == 1 - name: Ensure step-ca container is running - community.docker.docker_container: + containers.podman.podman_container: name: step-ca env: DOCKER_STEPCA_INIT_NAME: "Smallstep" DOCKER_STEPCA_INIT_DNS_NAMES: "localhost,{{ ansible_facts.nodename }},{{ ansible_facts.default_ipv4.address }}" DOCKER_STEPCA_INIT_REMOTE_MANAGEMENT: "true" DOCKER_STEPCA_INIT_ADDRESS: ":9999" - image: smallstep/step-ca - networks: - - name: host + image: docker.io/smallstep/step-ca + network: host volumes: - "step:/home/step" @@ -92,9 +64,13 @@ - health_result.status == 200 - health_result.json.status == "ok" + - name: Set step config path + set_fact: + step_config_path: "{{ ansible_facts.env.HOME }}/step" + - name: Stat provisioner password file ansible.builtin.stat: - path: /root/.step/provisioner-password + path: "{{ step_config_path }}/provisioner-password" register: provisioner_password_stat - name: Get provisioner password @@ -107,9 +83,9 @@ changed_when: false when: not provisioner_password_stat.stat.exists - - name: Create .step directory + - name: Create path for storing step password ansible.builtin.file: - path: /root/.step + path: "{{ step_config_path }}" state: directory mode: "0700" when: not provisioner_password_stat.stat.exists @@ -122,7 +98,7 @@ - name: Write provisioner password ansible.builtin.copy: content: "{{ provisioner_password.stdout }}" - dest: /root/.step/provisioner-password + dest: "{{ step_config_path }}/provisioner-password" mode: "0600" when: not provisioner_password_stat.stat.exists @@ -137,13 +113,11 @@ dest: "{{ step_ca_root_cert_local_path }}" mode: "0600" delegate_to: localhost - become: false - name: Install step CLI hosts: step tags: - step - become: true tasks: - name: Ensure step Deb is installed when: ansible_facts["os_family"] | lower == "debian" @@ -151,6 +125,7 @@ deb: "https://dl.smallstep.com/gh-release/cli/docs-cli-install/v0.24.4/step-cli_0.24.4_amd64.deb" state: present update_cache: true + become: true - name: Ensure step RPM is installed when: ansible_facts["os_family"] | lower == "redhat" @@ -161,14 +136,19 @@ # Package step-cli_0.24.4_amd643z16ickc.rpm is not signed disable_gpg_check: true state: present + become: true - name: Test step ansible.builtin.command: step certificate inspect https://smallstep.com changed_when: false + - name: Set step config path + set_fact: + step_config_path: "{{ ansible_facts.env.HOME }}/step" + - name: Regenerate step config if requested ansible.builtin.file: - path: "/root/.step/{{ item }}" + path: "{{ step_config_path }}/{{ item }}" state: absent loop: - certs @@ -177,7 +157,7 @@ - name: Check whether step has been bootstrapped ansible.builtin.stat: - path: /root/.step/config/defaults.json + path: "{{ step_config_path }}/config/defaults.json" register: step_stat - name: Get CA fingerprint # noqa: run-once[task] @@ -187,24 +167,88 @@ delegate_to: "{{ groups['step-ca'][0] }}" run_once: true + # Running an unprivileged step will prompt to overwrite the CA unless we --force + # This writes config under the unprivileged deployment user's HOME directory + # and previously (when we ran privileged) would be coupled with --install + # to write the CA to the system truststore - name: Bootstrap CA ansible.builtin.command: > step ca bootstrap --ca-url https://{{ hostvars[groups['step-ca'][0]].ansible_facts.default_ipv4.address }}:9999 - --fingerprint {{ ca_fingerprint.stdout }} --install + --fingerprint {{ ca_fingerprint.stdout }} --force changed_when: true when: not step_stat.stat.exists + - name: Determine step path when executed unprivileged + ansible.builtin.command: step path + changed_when: false + register: reductionist_step_path + + # Install CA from unprivileged deployment user's config to system truststore - name: Install root certificate to system - ansible.builtin.shell: step certificate install $(step path)/certs/root_ca.crt + ansible.builtin.shell: step certificate install {{ reductionist_step_path.stdout }}/certs/root_ca.crt changed_when: false + become: true when: not step_stat.stat.exists + # + # The following were originally Reductionist tasks + # + + - name: Check whether certificate exists + ansible.builtin.stat: + path: "{{ reductionist_remote_certs_path }}/cert.pem" + register: reductionist_cert_stat + + - name: Ensure remote certificate path exists + ansible.builtin.file: + path: "{{ reductionist_remote_certs_path }}" + state: directory + mode: "0711" + + - name: Generate a step token + ansible.builtin.command: >- + step ca token + --provisioner-password-file {{ step_config_path }}/provisioner-password + {{ reductionist_host }} + delegate_to: "{{ groups['step-ca'][0] }}" + changed_when: false + register: reductionist_step_token + + - name: Generate an initial certificate + ansible.builtin.command: >- + step ca certificate + --token {{ reductionist_step_token.stdout }} + --not-after {{ reductionist_cert_not_after }} + --force + {{ reductionist_host }} + {{ reductionist_remote_certs_path }}/cert.pem + {{ reductionist_remote_certs_path }}/key.pem + changed_when: true + when: not reductionist_cert_stat.stat.exists + + - name: Ensure certificate renewal systemd units exist + tags: privileged + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + mode: "0600" + loop: + - reductionist-cert-renewer.service + - reductionist-cert-renewer.timer + become: true + + - name: Ensure certificate renewal systemd timer is enabled + tags: privileged + ansible.builtin.service: + name: reductionist-cert-renewer.timer + enabled: true + become: true + - name: Deploy Minio hosts: minio tags: - minio - become: true tasks: - name: Assert that there is only one Minio server ansible.builtin.assert: @@ -212,13 +256,12 @@ groups['minio'] | length == 1 - name: Ensure minio container is running - community.docker.docker_container: + containers.podman.podman_container: name: minio-server - command: server data --console-address ":9001" - image: minio/minio - keep_volumes: false - networks: - - name: host + command: server data --console-address :9001 + image: docker.io/minio/minio + delete_volumes: true + network: host volumes: /data - name: Wait for minio object storage to start @@ -268,21 +311,20 @@ register: prometheus_yml - name: Ensure prometheus container is running - community.docker.docker_container: + containers.podman.podman_container: name: prometheus - image: prom/prometheus - networks: - - name: host + image: docker.io/prom/prometheus + network: host restart: "{{ prometheus_yml is changed or prometheus_cacert is changed }}" volumes: - "/etc/prometheus:/etc/prometheus:ro" - "prometheus:/prometheus" + become: false - name: Deploy Jaeger hosts: jaeger tags: - jaeger - become: true tasks: - name: Assert that there is only one Jaeger server ansible.builtin.assert: @@ -290,13 +332,12 @@ - groups['jaeger'] | length == 1 - name: Ensure jaeger container is running - community.docker.docker_container: + containers.podman.podman_container: name: jaeger env: COLLECTOR_ZIPKIN_HTTP_PORT: "9411" - image: jaegertracing/all-in-one:1.6 - networks: - - name: host + image: docker.io/jaegertracing/all-in-one:1.6 + network: host - name: Gather facts for Reductionist hosts: @@ -311,54 +352,7 @@ hosts: reductionist tags: - reductionist - become: true tasks: - - name: Check whether certificate exists - ansible.builtin.stat: - path: "{{ reductionist_remote_certs_path }}/cert.pem" - register: reductionist_cert_stat - - - name: Ensure remote certificate path exists - ansible.builtin.file: - path: "{{ reductionist_remote_certs_path }}" - state: directory - mode: "0700" - - - name: Generate a step token - ansible.builtin.command: >- - step ca token - --provisioner-password-file /root/.step/provisioner-password - {{ reductionist_host }} - delegate_to: "{{ groups['step-ca'][0] }}" - changed_when: false - register: reductionist_step_token - - - name: Generate an initial certificate - ansible.builtin.command: >- - step ca certificate - --token {{ reductionist_step_token.stdout }} - --not-after {{ reductionist_cert_not_after }} - --force - {{ reductionist_host }} - {{ reductionist_remote_certs_path }}/cert.pem - {{ reductionist_remote_certs_path }}/key.pem - changed_when: true - when: not reductionist_cert_stat.stat.exists - - - name: Ensure certificate renewal systemd units exist - ansible.builtin.template: - src: "{{ item }}.j2" - dest: "/etc/systemd/system/{{ item }}" - mode: "0600" - loop: - - reductionist-cert-renewer.service - - reductionist-cert-renewer.timer - - - name: Ensure certificate renewal systemd timer is enabled - ansible.builtin.service: - name: reductionist-cert-renewer.timer - enabled: true - - name: Clone reductionist repo ansible.builtin.git: repo: "{{ reductionist_src_url }}" @@ -367,21 +361,19 @@ when: reductionist_build_image | bool - name: Ensure reductionist image is built - community.docker.docker_image: + containers.podman.podman_image: name: "{{ reductionist_image }}" tag: "{{ reductionist_tag }}" - build: - network: host # Network to use for RUN cmds in dockerfile - needed to allow 'pip install...' in RedHat images - path: "{{ ansible_env.HOME }}/reductionist-rs" - source: build + path: "{{ ansible_env.HOME }}/reductionist-rs" when: reductionist_build_image | bool - name: Ensure reductionist container is running - community.docker.docker_container: + containers.podman.podman_container: name: "{{ reductionist_name }}" + privileged: true # Rocky 9 SELinux prevents visibility of the volume's certs otherwise env: "{{ reductionist_env }}" image: "{{ reductionist_image }}:{{ reductionist_tag }}" - networks: "{{ reductionist_networks }}" + network: "{{ reductionist_networks }}" volumes: "{{ reductionist_volumes }}" restart: true # Load new certificates. TODO: Hot reload @@ -429,14 +421,14 @@ register: haproxy_cfg - name: Ensure haproxy container is running - community.docker.docker_container: + containers.podman.podman_container: name: haproxy - image: haproxy:2.8 - networks: - - name: host + image: docker.io/haproxy:2.8 + network: host restart: "{{ haproxy_cfg is changed }}" volumes: - "/etc/haproxy:/usr/local/etc/haproxy:ro" + become: false - name: Wait for reductionist server to be accessible via HAProxy ansible.builtin.uri: diff --git a/docs/deployment.md b/docs/deployment.md index bfda1da..f385657 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -5,7 +5,7 @@ The Ansible playbook allows for a secure, scale-out deployment of Reductionist, The following services are supported: -* Docker engine +* Podman engine * Step CA Certificate Authority (generates certificates for Reductionist) * Step CLI (requests and renews certificates) * Minio object store (optional, for testing) @@ -18,11 +18,11 @@ The following services are supported: The existence of correctly configured hosts is assumed by this playbook. -The following host OS distributions are supported: +The following host OS distributions have been tested and are supported: -* Ubuntu 20.04-22.04 -* CentOS Stream 8-9 -* Rocky Linux 8-9 +* CentOS Stream 9 +* Rocky Linux 9 +* Ubuntu 24.04 Currently only a single network is supported. Several TCP ports should be accessible on this network. @@ -82,7 +82,7 @@ reductionist1 reductionist # Do not edit. -[docker:children] +[podman:children] haproxy jaeger minio @@ -133,21 +133,38 @@ ansible-galaxy collection install -r deployment/requirements.yml ## Deployment -Run the playbook: +Podman will be used to run containers under the same user account used for ansible deployment. +To install requisite system packages some tasks will require sudo `privileged` access. + +To run the entire playbook as an unprivileged user prompting for a sudo password: ```sh -ansible-playbook -i deployment/inventory deployment/site.yml +ansible-playbook -i deployment/inventory deployment/site.yml -K ``` -If you want to run only specific plays in the playbook, the following tags are supported and may be specified via `--tags `: +To run specific plays the following tags are supported and may be specified via `--tags `: -* `docker` +* `podman` - runs privileged tasks * `step-ca` -* `step` +* `step` - runs privileged tasks * `minio` -* `prometheus` +* `prometheus` - runs privileged tasks * `jaeger` * `reductionist` -* `haproxy` +* `haproxy` - runs privileged tasks + +### Minimal deployment of Podman and the Reductionist + +Podman is a prerequisite for running the Reductionist. + +Optionally run the `podman` play to install this prerequisite as an **unprivileged** user, the following will prompt for the sudo password to escalate privileges only for package installation: +```sh +ansible-playbook -i deployment/inventory deployment/site.yml --tags podman -K +``` + +Then to run the `reductionist` play, again as the **unprivileged** user: +```sh +ansible-playbook -i deployment/inventory deployment/site.yml --tags reductionist +``` ## Usage From 528d532350eeb7909ef0d64eb19aedfb6a26549d Mon Sep 17 00:00:00 2001 From: Max Norton Date: Thu, 13 Feb 2025 13:58:15 +0000 Subject: [PATCH 2/2] Update the podman installation play to enable linger on the unprivileged user account that'll be running the Reductionist, and any other, containers. We need linger enabled for the podman process to continue running after the user's logged out of the session. Deployment documentation updated. --- deployment/site.yml | 19 ++++++++++++++++++- docs/deployment.md | 16 +++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/deployment/site.yml b/deployment/site.yml index 9e0e0ef..2bf65b1 100644 --- a/deployment/site.yml +++ b/deployment/site.yml @@ -5,8 +5,22 @@ hosts: podman tags: - podman - become: true tasks: + # Unprivileged user account requires linger to be enabled so podman will continue to run + # after the session has been terminated upon logout + # The command: loginctl show-user + # Should show: Linger=yes + # When enabled we should have the file "/var/lib/systemd/linger/ + - name: Determine linger state for Reductionist user + ansible.builtin.stat: + path: "/var/lib/systemd/linger/{{ ansible_facts['user_id'] }}" + register: systemd_linger_path_for_reductionist + - name: Enable linger for unprivileged Reductionist user + ansible.builtin.command: loginctl enable-linger {{ ansible_facts['user_id'] }} + changed_when: true + when: not systemd_linger_path_for_reductionist.stat.exists + become: true + - name: Ensure podman docker is installed on Ubuntu when: ansible_facts["os_family"] | lower == "debian" ansible.builtin.package: @@ -15,6 +29,7 @@ - podman-docker state: present update_cache: true + become: true - name: Ensure podman docker is installed on RedHat when: ansible_facts["os_family"] | lower == "redhat" @@ -24,6 +39,7 @@ - podman-docker state: present update_cache: true + become: true - name: Ensure other system packages are present ansible.builtin.package: @@ -31,6 +47,7 @@ - git - python3-pip state: present + become: true - name: Deploy step CA hosts: step-ca diff --git a/docs/deployment.md b/docs/deployment.md index f385657..85bab59 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -155,8 +155,14 @@ To run specific plays the following tags are supported and may be specified via ### Minimal deployment of Podman and the Reductionist Podman is a prerequisite for running the Reductionist. +Podman can run containers as an **unprivileged** user, however this user must have **linger** enabled on their account to allow Podman to continue to run after logging out of the user session. -Optionally run the `podman` play to install this prerequisite as an **unprivileged** user, the following will prompt for the sudo password to escalate privileges only for package installation: +To enable **linger** support for the unprivileged user: +```sh +sudo loginctl enable-linger +``` + +Alternatively, run the optional `podman` play to install Podman as an **unprivileged** user. The following will prompt for the sudo password to escalate privileges only for package installation and for enabling **linger** for the unprivileged user: ```sh ansible-playbook -i deployment/inventory deployment/site.yml --tags podman -K ``` @@ -166,6 +172,14 @@ Then to run the `reductionist` play, again as the **unprivileged** user: ansible-playbook -i deployment/inventory deployment/site.yml --tags reductionist ``` +Podman containers require a manual restart after a system reboot. +This requires logging into the host(s) running the Reductionist as the **unprivileged** user to run: +```sh +podman restart reductionist +``` + +Automatic restart on boot can be enabled via **systemd**, not covered by this documentation. + ## Usage Once deployed, the Reductionist API is accessible on port 8080 by HAProxy. The Prometheus UI is accessible on port 9090 on the host running Prometheus. The Jaeger UI is accessible on port 16686 on the host running Jaeger.